{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DA3BF7A274C9420984E6233A09460048",
    "mdEditEnable": false
   },
   "source": [
    "# 导包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "96EE9D8C953142D7AEE81B639EA3DF92",
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000000 ./predictions/final_submission_20190811.csv\n\n1,1,0.53125741515603\n1,2,0.4392272420196072\n1,3,0.5879643281299144\n1,4,0.576160912203239\n1,5,0.5052100835619331\n1,6,0.4324311875604571\n1,7,0.4066308458541816\n2,1,0.5768602948272425\n2,2,0.6554118653967363\n2,3,0.5325662217303531\n1,128 770 122 1192,1,770 36 1192 8 33 10048 122 193 469 31 37\t\n1,128 770 122 1192,2,354 770 1192 40 9315 15 3545 14 522 3159 122 1645 4626 31 37\t\n1,128 770 122 1192,3,770 69380 1845 34 644 115 10320 11843 12348\t\n1,128 770 122 1192,4,770 770 4241 2131 7036 122 1192 851 2507\t\n1,128 770 122 1192,5,770 14692 3933 3194 27 10320 128 4346 1192\t\n1,128 770 122 1192,6,770 36 39 1192 266 108 5229 192 122 26831 11\t\n1,128 770 122 1192,7,770 36 10048 122 13 449 1192 33 708 30 37\t\n2,66 64 123 848,1,3589 458 848 66 26 123 848 388 17779 29794 27 1282 2435 389\t\n2,66 64 123 848,2,3589 458 848 66 26 123 848 388 17779 328 274 82424\t\n2,66 64 123 848,3,5218 1257 31 4712 6030 27 47 267 659 1392 3589 66 64 123 27 5692 848 819\t\n\n10318082,8,0.26414565025898734\n10318082,9,0.42693966127920707\n10318082,10,0.28341074642672387\n10318082,11,0.2947948388734799\n10318082,12,0.26474696779822326\n10318082,13,0.32344879648630276\n10318082,14,0.34928164192293326\n10318082,15,0.2033738233624376\n10318082,16,0.055942568163945425\n10318082,17,0.19353918772727668\n10318082,794 39322,8,14703 82 16 21 258 13958 15 39322 27 15488 794 91154 1075 39322 148 4633\t\n10318082,794 39322,9,19352 1424 9240 203 11529 27 6526 4452 19398 4813 733\t\n10318082,794 39322,10,23860 68 9 220 235 161 16 794 39410 1151 2448\t\n10318082,794 39322,11,37314 80 180 11 794 20 915 102 1424 9560 35 9240 31420 936 36 10939 236 2947\t\n10318082,794 39322,12,9683 2165 817 3541 1424 39322 35 55495 15 69\t\n10318082,794 39322,13,49 118 1424 39322 3909 1351 27 196 151 1585 15 52\t\n10318082,794 39322,14,8633 18020 83 262 11 1383 1424 154 679 5760 27 2860 2251 281\t\n10318082,794 39322,15,8633 25659 1121 1424 9240 10773 27 1851 24 47965 23\t\n10318082,794 39322,16,8633 18866 98 3526 624 85 527 31 27 124 2150 281\t\n10318082,794 39322,17,16001 145 794 75 2597 1424 35 10960 39 38276 2030\t\n"
     ]
    }
   ],
   "source": [
    "!wc -l ./predictions/final_submission_20190811.csv\n",
    "print()\n",
    "!head ./predictions/final_submission_20190811.csv\n",
    "!head /home/kesci/input/bytedance/bytedance_contest.final_2.csv\n",
    "print()\n",
    "!tail ./predictions/final_submission_20190811.csv\n",
    "!tail /home/kesci/input/bytedance/bytedance_contest.final_2.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "24CDE881EEC648168F273287B4921514",
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting fuzzywuzzy\n  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/d8/f1/5a267addb30ab7eaa1beab2b9323073815da4551076554ecc890a3595ec9/fuzzywuzzy-0.17.0-py2.py3-none-any.whl\nInstalling collected packages: fuzzywuzzy\nSuccessfully installed fuzzywuzzy-0.17.0\nCollecting jellyfish\n  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/3f/80/bcacc7affb47be7279d7d35225e1a932416ed051b315a7f9df20acf04cbe/jellyfish-0.7.2.tar.gz (133kB)\n\u001b[K    100% |████████████████████████████████| 143kB 7.6MB/s eta 0:00:01\n\u001b[?25hBuilding wheels for collected packages: jellyfish\n  Running setup.py bdist_wheel for jellyfish ... \u001b[?25ldone\n\u001b[?25h  Stored in directory: /home/kesci/.cache/pip/wheels/bc/b7/78/6736d761d7635d2af9579e040f342b5482850d856d26cbefa3\nSuccessfully built jellyfish\nInstalling collected packages: jellyfish\nSuccessfully installed jellyfish-0.7.2\nCollecting pyemd\n  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/c0/c5/7fea8e7a71cd026b30ed3c40e4c5ea13a173e28f8855da17e25271e8f545/pyemd-0.5.1.tar.gz (91kB)\n\u001b[K    100% |████████████████████████████████| 92kB 7.5MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied: numpy<2.0.0,>=1.9.0 in /opt/conda/lib/python3.6/site-packages (from pyemd)\nBuilding wheels for collected packages: pyemd\n  Running setup.py bdist_wheel for pyemd ... \u001b[?25ldone\n\u001b[?25h  Stored in directory: /home/kesci/.cache/pip/wheels/94/20/c1/ccdf0e9878f5c76def850603e62a572746036829f8353804bd\nSuccessfully built pyemd\nInstalling collected packages: pyemd\nSuccessfully installed pyemd-0.5.1\nCollecting python-levenshtein\n  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/42/a9/d1785c85ebf9b7dfacd08938dd028209c34a0ea3b1bcdb895208bd40a67d/python-Levenshtein-0.12.0.tar.gz (48kB)\n\u001b[K    100% |████████████████████████████████| 51kB 5.9MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied: setuptools in /opt/conda/lib/python3.6/site-packages (from python-levenshtein)\nBuilding wheels for collected packages: python-levenshtein\n  Running setup.py bdist_wheel for python-levenshtein ... \u001b[?25ldone\n\u001b[?25h  Stored in directory: /home/kesci/.cache/pip/wheels/ef/af/8f/b3250804480b8d14ca55d436129a2fb53798a0ae9287b686c0\nSuccessfully built python-levenshtein\nInstalling collected packages: python-levenshtein\nSuccessfully installed python-levenshtein-0.12.0\n"
     ]
    }
   ],
   "source": [
    "# !pip install fuzzywuzzy -i https://pypi.tuna.tsinghua.edu.cn/simple\n",
    "# !pip install jellyfish -i https://pypi.tuna.tsinghua.edu.cn/simple\n",
    "# !pip install pyemd -i https://pypi.tuna.tsinghua.edu.cn/simple\n",
    "# !pip install python-levenshtein -i https://pypi.tuna.tsinghua.edu.cn/simple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cell_type": "code",
    "id": "A5B4D140EABC45748C3C9FDBD11352E9",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.3 s, sys: 596 ms, total: 1.9 s\nWall time: 1.45 s\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "\r\n",
    "import os\r\n",
    "import sys\r\n",
    "import math\r\n",
    "import pickle\r\n",
    "import collections\r\n",
    "import gc\r\n",
    "import joblib\r\n",
    "import gzip\r\n",
    "import xgboost as xgb\r\n",
    "import seaborn as sns\r\n",
    "import matplotlib.pyplot as plt\r\n",
    "import pandas as pd\r\n",
    "import numpy as np\r\n",
    "import lightgbm as lgb\r\n",
    "from tqdm import tqdm\r\n",
    "# from fuzzywuzzy import fuzz\r\n",
    "from datetime import datetime\r\n",
    "from multiprocessing import Pool\r\n",
    "# from jellyfish import jaro_distance, jaro_winkler\r\n",
    "from scipy.spatial.distance import cosine, euclidean, cityblock\r\n",
    "\r\n",
    "import gensim\r\n",
    "from gensim.corpora import Dictionary\r\n",
    "from gensim.models import TfidfModel, FastText, KeyedVectors\r\n",
    "from gensim.models.word2vec import Word2Vec, PathLineSentences, LineSentence\r\n",
    "\r\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, KFold\r\n",
    "from sklearn.metrics import roc_auc_score, precision_score\r\n",
    "from sklearn.ensemble import RandomForestClassifier\r\n",
    "from sklearn.externals import joblib\r\n",
    "\r\n",
    "import networkx as nx\r\n",
    "from networkx.readwrite.gpickle import write_gpickle, read_gpickle\r\n",
    "\r\n",
    "import warnings\r\n",
    "warnings.filterwarnings('ignore', category=Warning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CB3D344A4EC943C0A0C15B9A2A12B5D2",
    "mdEditEnable": false
   },
   "source": [
    "# 常量初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "3F0E11F980C4407BBE75220A36839EB2",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\nWall time: 5.72 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "test_data_path = '/home/kesci/input/bytedance/test_final_part1.csv'\r\n",
    "train_data_path = './split/10kw.csv'\r\n",
    "final_data_path = '/home/kesci/input/bytedance/bytedance_contest.final_2.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "A79292BD4C6743DB87694C0BF6317C01",
    "mdEditEnable": false
   },
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2AC4F400337042CD9995DFAA8F54ED87",
    "mdEditEnable": false
   },
   "source": [
    "## 获取query和title的set集合——train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "0F3B4DE0DBCE4646A79321325D56F5B3",
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\nWall time: 10.3 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "def get_unique(filepath):\r\n",
    "    print('-----------------reading {}-----------------'.format(filepath))\r\n",
    "    data = pd.read_csv(filepath, chunksize=1000000, header=None)\r\n",
    "    data.columns = {'query_id', 'query', 'title_id', 'title', 'label'}\r\n",
    "    sentences = set()\r\n",
    "    query = set()\r\n",
    "    title = set()\r\n",
    "    print('------------------parse data------------------')\r\n",
    "    i = 0\r\n",
    "    for chunk in data:\r\n",
    "        if i%10 == 0: print(i)\r\n",
    "        i+=1\r\n",
    "        query |= set(chunk[1].unique().tolist())\r\n",
    "        title |= set(chunk[3].unique().tolist())\r\n",
    "    sentences = query | title\r\n",
    "    \r\n",
    "    print('unique query size', query.__len__())\r\n",
    "    print('unique title size', title.__len__())\r\n",
    "    print('unique sentences size', sentences.__len__())\r\n",
    "    # print('---------------Writing to csv file---------------')\r\n",
    "\r\n",
    "    # with open(handled_path + 'sentences.csv', 'w', encoding='utf-8') as f:\r\n",
    "    #     for item in sentences:\r\n",
    "    #         f.write(item)\r\n",
    "    #         f.write('\\n')\r\n",
    "    return query, title, sentences\r\n",
    "test_query, test_title, test_sentence = get_unique(test_data_path)\r\n",
    "train_query, train_title, train_sentence = get_unique(train_data_path)\r\n",
    "final_query, final_title, final_sentence = get_unique(final_data_path)\r\n",
    "\r\n",
    "sentence = test_sentence | train_sentence | final_sentence\r\n",
    "print(sentence.__len__())\r\n",
    "with open('./handled_data/all_sentences.csv', 'w', encoding='utf-8') as f:\r\n",
    "    for item in sentence:\r\n",
    "        f.write(item)\r\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "F4AA3A285E81414588A75871F76BADC1",
    "mdEditEnable": false
   },
   "source": [
    "## wordcount"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "702619C5323F48EB86F7B69F60054BB0"
   },
   "outputs": [],
   "source": [
    "word_counter = collections.Counter()\r\n",
    "\r\n",
    "def word_count(filepath):\r\n",
    "    data = pd.read_csv(filepath, chunksize=1000000, header=None)\r\n",
    "    for chunk in tqdm(data):\r\n",
    "        for item in chunk[1].unique():\r\n",
    "            for word in item.split():\r\n",
    "                word_counter[word] += 1\r\n",
    "        for item in chunk[3].tolist():\r\n",
    "            for word in item.split():\r\n",
    "                word_counter[word] += 1\r\n",
    "                \r\n",
    "def get_weight(count, eps=10000, min_count=2):\r\n",
    "    return 0 if count < min_count else 1/(count + eps)\r\n",
    "    \r\n",
    "word_count(test_data_path)\r\n",
    "word_count(train_data_path)\r\n",
    "word_count(final_data_path)\r\n",
    "\r\n",
    "weights = {word : get_weight(count) for word, count in word_counter.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "816FB6F2013E4F7187CD3AC4DCECFC2A"
   },
   "outputs": [],
   "source": [
    "def word_shares(row):\r\n",
    "    q1_list = str(row['question1']).lower().split()\r\n",
    "    q1 = set(q1_list)\r\n",
    "    q1words = q1.difference(stops)\r\n",
    "    if len(q1words) == 0:\r\n",
    "        return '0:0:0:0:0:0:0:0'\r\n",
    "\r\n",
    "    q2_list = str(row['question2']).lower().split()\r\n",
    "    q2 = set(q2_list)\r\n",
    "    q2words = q2.difference(stops)\r\n",
    "    if len(q2words) == 0:\r\n",
    "        return '0:0:0:0:0:0:0:0'\r\n",
    "\r\n",
    "    words_hamming = sum(1 for i in zip(q1_list, q2_list) if i[0]==i[1])/max(len(q1_list), len(q2_list))\r\n",
    "\r\n",
    "    q1stops = q1.intersection(stops)\r\n",
    "    q2stops = q2.intersection(stops)\r\n",
    "\r\n",
    "    q1_2gram = set([i for i in zip(q1_list, q1_list[1:])])\r\n",
    "    q2_2gram = set([i for i in zip(q2_list, q2_list[1:])])\r\n",
    "\r\n",
    "    shared_2gram = q1_2gram.intersection(q2_2gram)\r\n",
    "\r\n",
    "    shared_words = q1words.intersection(q2words)\r\n",
    "    shared_weights = [weights.get(w, 0) for w in shared_words]\r\n",
    "    q1_weights = [weights.get(w, 0) for w in q1words]\r\n",
    "    q2_weights = [weights.get(w, 0) for w in q2words]\r\n",
    "    total_weights = q1_weights + q1_weights\r\n",
    "\r\n",
    "    R1 = np.sum(shared_weights) / np.sum(total_weights) #tfidf share\r\n",
    "    R2 = len(shared_words) / (len(q1words) + len(q2words) - len(shared_words)) #count share\r\n",
    "    R31 = len(q1stops) / len(q1words) #stops in q1\r\n",
    "    R32 = len(q2stops) / len(q2words) #stops in q2\r\n",
    "    Rcosine_denominator = (np.sqrt(np.dot(q1_weights,q1_weights))*np.sqrt(np.dot(q2_weights,q2_weights)))\r\n",
    "    Rcosine = np.dot(shared_weights, shared_weights)/Rcosine_denominator\r\n",
    "    if len(q1_2gram) + len(q2_2gram) == 0:\r\n",
    "        R2gram = 0\r\n",
    "    else:\r\n",
    "        R2gram = len(shared_2gram) / (len(q1_2gram) + len(q2_2gram))\r\n",
    "    return '{}:{}:{}:{}:{}:{}:{}:{}'.format(R1, R2, len(shared_words), R31, R32, R2gram, Rcosine, words_hamming)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "58E84E29DB9A4779B04D1D0608625D1C",
    "mdEditEnable": false
   },
   "source": [
    "# 生成无向图"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "48483DE7E283464194017A7FC7AA040D",
    "mdEditEnable": false
   },
   "source": [
    "## 准备数据——无向图生成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "EFC4238523B94860B94B2C0B1E78E2A4",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start reading seq2id\nend reading seq2id\nseq2id with length ---> 87907776\nCPU times: user 1min 31s, sys: 14.5 s, total: 1min 45s\nWall time: 2min 4s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print('start reading seq2id')\n",
    "seq2id = {}\n",
    "with open('./handled_data/all_sentences.csv', 'r', encoding='utf-8') as f:\n",
    "    for i, sentence in enumerate(f.readlines()):\n",
    "        seq2id[sentence.strip()] = i\n",
    "print('end reading seq2id')\n",
    "print('seq2id with length --->',len(seq2id))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7377F073719A4D9E85A8E2AC1BF9A80B",
    "mdEditEnable": false
   },
   "source": [
    "## 文本转ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "CE5D7FCDC3474DD5B96D9BE64D271CD3",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-04 18:25:18.316136\nstart transform data\n0\n10\n                  0             1\ncount  2.000000e+07  2.000000e+07\nmean   5.122025e+07  6.027587e+07\nstd    2.944711e+07  2.793264e+07\nmin    1.060000e+02  0.000000e+00\n25%    2.575488e+07  3.862078e+07\n50%    5.130830e+07  6.404053e+07\n75%    7.676274e+07  8.443098e+07\nmax    1.019757e+08  1.019757e+08\nCPU times: user 1min 18s, sys: 1.86 s, total: 1min 20s\nWall time: 1min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(datetime.now())\n",
    "def transform(filepath):\n",
    "    data = pd.read_csv(filepath, chunksize=1000000, header=None)\n",
    "    data.columns = {'query_id', 'query', 'title_id', 'title', 'label'}\n",
    "    result = []\n",
    "    print('start transform data')\n",
    "    idx = 0\n",
    "    for chunk in data:\n",
    "        if idx%10==0: print(idx)\n",
    "        idx += 1\n",
    "        result.append(pd.DataFrame({0:chunk[1].apply(lambda x : seq2id[x.strip()]), \n",
    "                                    1:chunk[3].apply(lambda x : seq2id[x.strip()])}))\n",
    "    return result\n",
    "# id = pd.concat(transform(test_data_path))\n",
    "print(id.describe())\n",
    "id.to_csv('./handled_data/id_test.csv', header=None, index=False)\n",
    "!wc -l ./handled_data/id_test.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AA79B229CB7A4F4E9E91B07BFE51D406",
    "mdEditEnable": false
   },
   "source": [
    "## 生成无向图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "5A8E5EEB63F44D558A48483A78E43597",
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-04 20:09:48.426697\ngenerate train graph\n2019-08-04 20:10:11.280335 ---> 0\n2019-08-04 20:10:16.641768 ---> 1\n2019-08-04 20:10:21.355543 ---> 2\n2019-08-04 20:10:27.104319 ---> 3\n2019-08-04 20:10:31.788189 ---> 4\n2019-08-04 20:10:36.931844 ---> 5\n2019-08-04 20:10:40.341101 ---> 6\n2019-08-04 20:10:46.926877 ---> 7\n2019-08-04 20:10:53.250733 ---> 8\n2019-08-04 20:10:56.573122 ---> 9\n2019-08-04 20:10:59.920231 ---> 10\n2019-08-04 20:11:07.224468 ---> 11\n2019-08-04 20:11:10.635839 ---> 12\n2019-08-04 20:11:14.068377 ---> 13\n2019-08-04 20:11:24.505138 ---> 14\n2019-08-04 20:11:27.838067 ---> 15\n2019-08-04 20:11:31.123212 ---> 16\n2019-08-04 20:11:34.418728 ---> 17\n2019-08-04 20:11:44.396497 ---> 18\n2019-08-04 20:11:47.722042 ---> 19\n2019-08-04 20:11:51.068181 ---> 20\n2019-08-04 20:11:54.431964 ---> 21\n2019-08-04 20:11:57.809854 ---> 22\n2019-08-04 20:12:10.010839 ---> 23\n2019-08-04 20:12:13.399495 ---> 24\n2019-08-04 20:12:16.806213 ---> 25\n2019-08-04 20:12:20.229955 ---> 26\n2019-08-04 20:12:23.646422 ---> 27\n2019-08-04 20:12:27.061730 ---> 28\n2019-08-04 20:12:30.500476 ---> 29\nfinish generate train graph, temporaray graph size -> 21236030 29997376\ngenerate final data graph\n2019-08-04 20:13:01.251764 ---> 0\n2019-08-04 20:13:04.778456 ---> 1\n2019-08-04 20:13:12.256328 ---> 2\n2019-08-04 20:13:15.593915 ---> 3\n2019-08-04 20:13:18.930359 ---> 4\n2019-08-04 20:13:22.265382 ---> 5\n2019-08-04 20:13:25.573647 ---> 6\n2019-08-04 20:13:28.886724 ---> 7\n2019-08-04 20:13:32.212384 ---> 8\n2019-08-04 20:13:50.524109 ---> 9\n2019-08-04 20:13:53.876998 ---> 10\n2019-08-04 20:13:57.215895 ---> 11\n2019-08-04 20:14:00.570448 ---> 12\n2019-08-04 20:14:03.952938 ---> 13\n2019-08-04 20:14:07.322928 ---> 14\n2019-08-04 20:14:10.687618 ---> 15\n2019-08-04 20:14:14.065740 ---> 16\n2019-08-04 20:14:17.448141 ---> 17\n2019-08-04 20:14:20.841102 ---> 18\n2019-08-04 20:14:24.321381 ---> 19\n2019-08-04 20:14:27.720988 ---> 20\n2019-08-04 20:14:31.123869 ---> 21\n2019-08-04 20:14:34.535073 ---> 22\n2019-08-04 20:14:58.253944 ---> 23\n2019-08-04 20:15:01.721445 ---> 24\n2019-08-04 20:15:05.162061 ---> 25\n2019-08-04 20:15:08.601864 ---> 26\n2019-08-04 20:15:12.042046 ---> 27\n2019-08-04 20:15:15.479877 ---> 28\n2019-08-04 20:15:18.911179 ---> 29\n2019-08-04 20:15:22.363532 ---> 30\n2019-08-04 20:15:25.815807 ---> 31\n2019-08-04 20:15:29.282121 ---> 32\n2019-08-04 20:15:32.731438 ---> 33\n2019-08-04 20:15:36.196011 ---> 34\n2019-08-04 20:15:39.658354 ---> 35\n2019-08-04 20:15:43.136249 ---> 36\n2019-08-04 20:15:46.602263 ---> 37\n2019-08-04 20:15:50.072394 ---> 38\n2019-08-04 20:15:53.564058 ---> 39\n2019-08-04 20:15:57.031866 ---> 40\n2019-08-04 20:16:00.521455 ---> 41\n2019-08-04 20:16:04.020592 ---> 42\n2019-08-04 20:16:07.518894 ---> 43\n2019-08-04 20:16:39.318845 ---> 44\n2019-08-04 20:16:42.826865 ---> 45\n2019-08-04 20:16:46.338864 ---> 46\n2019-08-04 20:16:49.844742 ---> 47\n2019-08-04 20:16:53.361940 ---> 48\n2019-08-04 20:16:56.883326 ---> 49\n2019-08-04 20:17:00.410362 ---> 50\n2019-08-04 20:17:03.951037 ---> 51\n2019-08-04 20:17:07.481663 ---> 52\n2019-08-04 20:17:18.732522 ---> 53\n2019-08-04 20:17:22.134094 ---> 54\n2019-08-04 20:17:25.447526 ---> 55\n2019-08-04 20:17:28.776571 ---> 56\n2019-08-04 20:17:32.107264 ---> 57\n2019-08-04 20:17:35.436777 ---> 58\n2019-08-04 20:17:38.796158 ---> 59\n2019-08-04 20:17:42.136024 ---> 60\n2019-08-04 20:17:45.490189 ---> 61\n2019-08-04 20:17:48.824860 ---> 62\n2019-08-04 20:17:52.182171 ---> 63\n2019-08-04 20:17:55.530998 ---> 64\n2019-08-04 20:17:58.895756 ---> 65\n2019-08-04 20:18:02.289543 ---> 66\n2019-08-04 20:18:05.658089 ---> 67\n2019-08-04 20:18:09.042868 ---> 68\n2019-08-04 20:18:12.423860 ---> 69\n2019-08-04 20:18:15.742892 ---> 70\n2019-08-04 20:18:19.046646 ---> 71\n2019-08-04 20:18:22.362017 ---> 72\n2019-08-04 20:18:25.699028 ---> 73\n2019-08-04 20:18:29.022436 ---> 74\n2019-08-04 20:18:32.372405 ---> 75\n2019-08-04 20:19:16.084752 ---> 76\n2019-08-04 20:19:19.441777 ---> 77\n2019-08-04 20:19:22.803283 ---> 78\n2019-08-04 20:19:26.165686 ---> 79\n2019-08-04 20:19:29.536950 ---> 80\n2019-08-04 20:19:32.917675 ---> 81\n2019-08-04 20:19:36.283755 ---> 82\n2019-08-04 20:19:39.648486 ---> 83\n2019-08-04 20:19:43.019010 ---> 84\n2019-08-04 20:19:46.380202 ---> 85\n2019-08-04 20:19:49.725886 ---> 86\n2019-08-04 20:19:53.097143 ---> 87\n2019-08-04 20:19:56.476532 ---> 88\n2019-08-04 20:19:59.839292 ---> 89\n2019-08-04 20:20:03.241092 ---> 90\n2019-08-04 20:20:06.634720 ---> 91\n2019-08-04 20:20:10.003162 ---> 92\n2019-08-04 20:20:13.364029 ---> 93\n2019-08-04 20:20:16.724924 ---> 94\n2019-08-04 20:20:20.098426 ---> 95\n2019-08-04 20:20:23.469628 ---> 96\n2019-08-04 20:20:26.833314 ---> 97\n2019-08-04 20:20:30.207836 ---> 98\n2019-08-04 20:20:33.582591 ---> 99\nfinish generate final graph, temporaray graph size -> 59028158 129984026\nsave graph\nCPU times: user 13min 57s, sys: 51.8 s, total: 14min 49s\nWall time: 15min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(datetime.now())\n",
    "G = nx.Graph()\n",
    "    \n",
    "def make_graph_parallel():\n",
    "    CHUNK_SIZE = 1000000\n",
    "    # test_data = pd.read_csv('./handled_data/id_test.csv', chunksize=CHUNK_SIZE, header=None)\n",
    "    # test_data.columns = {'query_id', 'query', 'title_id', 'title', 'label'}\n",
    "    \n",
    "    # print('generate test graph')\n",
    "    # idx = 0\n",
    "    # print(datetime.now())\n",
    "    # for chunk in test_data:\n",
    "    #     print(datetime.now(),'--->',idx)\n",
    "    #     idx += 1 \n",
    "    #     ziped = list(zip(chunk[0], chunk[1]))\n",
    "    #     G.add_edges_from(ziped)\n",
    "        \n",
    "    #     del chunk\n",
    "    #     del ziped\n",
    "    # print('finish generate test graph, temporaray graph size ->', G.number_of_nodes(), G.number_of_edges())\n",
    "    # del test_data\n",
    "    # gc.collect()\n",
    "    \n",
    "    data = pd.read_csv('./handled_data/id_train.csv', chunksize=CHUNK_SIZE, header=None)\n",
    "    data.columns = {'query_id', 'query', 'title_id', 'title', 'label'}\n",
    "\n",
    "    print('generate train graph')\n",
    "    idx = 0\n",
    "    # for chunk in data:\n",
    "    for i in range(100):\n",
    "        chunk = data.get_chunk()\n",
    "        if i < 70: continue\n",
    "        print(datetime.now(), '--->', idx)\n",
    "        idx += 1\n",
    "        ziped = list(zip(chunk[0], chunk[1]))\n",
    "        G.add_edges_from(ziped)\n",
    "        \n",
    "        del chunk\n",
    "        del ziped\n",
    "    del data\n",
    "    gc.collect()\n",
    "    print('finish generate train graph, temporaray graph size ->', G.number_of_nodes(), G.number_of_edges())\n",
    "    \n",
    "    data = pd.read_csv('./handled_data/id_final.csv', chunksize=CHUNK_SIZE, header=None)\n",
    "    data.columns = {'query_id', 'query', 'title_id', 'title', 'label'}\n",
    "    print('generate final data graph')\n",
    "    idx = 0\n",
    "    for chunk in data:\n",
    "        print(datetime.now(), '--->', idx)\n",
    "        idx += 1\n",
    "        ziped = list(zip(chunk[0], chunk[1]))\n",
    "        G.add_edges_from(ziped)\n",
    "        \n",
    "        del chunk\n",
    "        del ziped\n",
    "    print('finish generate final graph, temporaray graph size ->', G.number_of_nodes(), G.number_of_edges())\n",
    "   \n",
    "    del data\n",
    "    gc.collect()\n",
    "    print('save graph')\n",
    "    write_gpickle(G, './handled_data/final_graph.pkl')\n",
    "    \n",
    "make_graph_parallel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8D8EA63E6CA2463C9485424F7D635E46",
    "mdEditEnable": false
   },
   "source": [
    "## 生成pagerank模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "07291A2657994E6095D81705A142B993",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-04 20:26:20.708101\nend generate pagerank value\nCPU times: user 21min, sys: 51.1 s, total: 21min 52s\nWall time: 21min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "print(datetime.now())\r\n",
    "graph = read_gpickle('./handled_data/final_graph.pkl')\r\n",
    "print(datetime.now(),'---> end load graph')\r\n",
    "page_rank = nx.pagerank_scipy(graph)\r\n",
    "print('end generate pagerank value')\r\n",
    "pickle.dump(page_rank, open('./handled_data/pagerank_final.model', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "66D2DD90D6744BDF9ADD4D6E6588003C",
    "mdEditEnable": false
   },
   "source": [
    "## 生成HITS模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "A671B600F9154D908E108D9B14317892",
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-07-27 03:39:19.758518\nend load graph\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "print(datetime.datetime.now())\r\n",
    "graph = joblib.load('./models/un_direct_graph.model')\r\n",
    "print('end load graph')\r\n",
    "hits_h, hits_a = nx.hits(graph)\r\n",
    "print('end generate HITS value')\r\n",
    "joblib.dump(hits_h, './models/hub_value.model')\r\n",
    "joblib.dump(hits_a, './models/authority_value.model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0B5EB347F584409BB654A641F66B1B2A",
    "mdEditEnable": false
   },
   "source": [
    "## 数据准备——特征生成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "D9427953B24748D5892B3709E233DC89",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 52.5 s, sys: 12.3 s, total: 1min 4s\nWall time: 1min 5s\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "# graph = read_gpickle('./handled_data/final/final_graph.pkl')\r\n",
    "# print('end load graph')\r\n",
    "# seq2id = {}\r\n",
    "# with open('./handled_data/all_sentences_verseion_2.csv', 'r', encoding='utf-8') as f:\r\n",
    "#     for i, sentence in enumerate(f.readlines()):\r\n",
    "#         seq2id[sentence.strip()] = i\r\n",
    "# print('end seq2id')\r\n",
    "def gen_degrees():\r\n",
    "    max_degrees = {}\r\n",
    "    edges = graph.edges()\r\n",
    "    for edge in edges:\r\n",
    "        for n in edge:\r\n",
    "            max_degrees[n] = max_degrees.get(n, 0) + 1\r\n",
    "    return max_degrees\r\n",
    "\r\n",
    "def gen_components():\r\n",
    "    max_components = {}\r\n",
    "    components = nx.connected_components(graph)\r\n",
    "    for component in components:\r\n",
    "        for n in component:\r\n",
    "            max_components[n] = max(max_components.get(n, 0), len(component))\r\n",
    "    return max_components\r\n",
    "\r\n",
    "def gen_hits():\r\n",
    "    hits_h, hits_a = nx.hits(graph, max_iter=500)\r\n",
    "    return hits_h, hits_a\r\n",
    "\r\n",
    "# max_degrees = gen_degrees()\r\n",
    "# pickle.dump(max_degrees, open('./handled_data/max_degree_final', 'wb'))\r\n",
    "# print('end degree')\r\n",
    "# max_components = gen_components()\r\n",
    "# pickle.dump(max_components, open('./handled_data/max_components_final', 'wb'))\r\n",
    "# print('end components')\r\n",
    "# del graph\r\n",
    "# gc.collect()\r\n",
    "\r\n",
    "max_degrees = pickle.load(open('./handled_data/final/max_degree_final', 'rb'))\r\n",
    "max_components = pickle.load(open('./handled_data/final/max_components_final', 'rb'))\r\n",
    "page_rank = pickle.load(open('./handled_data/final/pagerank_final.model', 'rb'))\r\n",
    "# print('end pagerank')\r\n",
    "# hits_h, hits_a = gen_hits()\r\n",
    "\r\n",
    "# max_degree max_components\r\n",
    "def calculate_statistics(row):\r\n",
    "    return [max_degrees[row[0]], max_degrees[row[1]], max_components[row[0]]]      \r\n",
    "     \r\n",
    "# PageRank     \r\n",
    "def calculate_pagerank(row):\r\n",
    "    return [page_rank[row[0]] * 1e6, page_rank[row[1]] * 1e6]       \r\n",
    "\r\n",
    "# Hits\r\n",
    "def calculate_hits(row):\r\n",
    "    hits_h_1 = hits_h[row[0]] * 1e6\r\n",
    "    hits_a_1 = hits_a[row[0]] * 1e6\r\n",
    "    hits_h_2 = hits_h[row[1]] * 1e6\r\n",
    "    hits_a_2 = hits_a[row[1]] * 1e6\r\n",
    "    return [hits_h_1, hits_a_1, hits_h_2, hits_a_2]\r\n",
    "\r\n",
    "# ShortestPath\r\n",
    "def calculate_shortestpath(row):\r\n",
    "    graph.remove_edge(row[0], row[1])\r\n",
    "    if nx.has_path(graph, row[0], row[1]):\r\n",
    "        shortest_path = len(nx.shortest_path(graph, row[0], row[1]))\r\n",
    "    else:\r\n",
    "        shortest_path = -1\r\n",
    "    graph.add_edge(row[0], row[1])\r\n",
    "    return [shortest_path]\r\n",
    "\r\n",
    "# Neighbour\r\n",
    "def calculate_neighbour(row):\r\n",
    "    neighbor_1 = set(graph.neighbors(row[0]))\r\n",
    "    neighbor_2 = set(graph.neighbors(row[1]))\r\n",
    "    return [len(neighbor_1), len(neighbor_2), len(neighbor_1 | neighbor_2)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "049F004A503C405A85B47CBD208D9567",
    "mdEditEnable": false
   },
   "source": [
    "## 生成图特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "D36B363E31CD476C870BEDF3958CCD4A",
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start generate feature\n0\n1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25\n26\n27\n28\n29\n(30000000, 5)\n                  0             1             2             0             1\ncount  3.000000e+07  3.000000e+07  3.000000e+07  3.000000e+07  3.000000e+07\nmean   1.274293e+02  1.106532e+01  5.754290e+07  1.234310e+00  2.669189e-02\nstd    3.819820e+03  3.443744e+02  5.868692e+06  3.824809e+01  3.425827e+00\nmin    1.000000e+00  1.000000e+00  2.000000e+00  2.574202e-03  2.541272e-03\n25%    7.000000e+00  1.000000e+00  5.814144e+07  4.419803e-02  3.981151e-03\n50%    1.600000e+01  3.000000e+00  5.814144e+07  8.919203e-02  6.508193e-03\n75%    2.000000e+01  7.000000e+00  5.814144e+07  1.496774e-01  1.367823e-02\nmax    1.289980e+05  1.289980e+05  5.814144e+07  1.291623e+03  1.291623e+03\nCPU times: user 6min 29s, sys: 21.1 s, total: 6min 50s\nWall time: 7min 1s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\r\n",
    "gc.collect()\r\n",
    "CHUNKSIZE = 1000000\r\n",
    "data = pd.read_csv('./handled_data/id/id_train.csv', chunksize=CHUNKSIZE, header=None)\r\n",
    "data.columns = {'query_id', 'query', 'title_id', 'title', 'label'}\r\n",
    "pool = Pool()\r\n",
    "res = []\r\n",
    "idx = 0\r\n",
    "for i in range(70):\r\n",
    "    data.get_chunk()\r\n",
    "print('start generate feature')\r\n",
    "# for chunk in data:\r\n",
    "for i in range(30):\r\n",
    "    chunk = data.get_chunk()\r\n",
    "    print(idx)\r\n",
    "    idx += 1\r\n",
    "    df = []\r\n",
    "    ziped = list(zip(chunk[0], chunk[1]))\r\n",
    "    # ziped = list(zip(chunk[1].apply(lambda x:seq2id[x.strip()]).tolist(), \r\n",
    "    #                                 chunk[3].apply(lambda x:seq2id[x.strip()]).tolist()))\r\n",
    "                                    \r\n",
    "    df.append(pd.DataFrame(pool.map(calculate_statistics, ziped)))\r\n",
    "    df.append(pd.DataFrame(pool.map(calculate_pagerank, ziped)))\r\n",
    "    \r\n",
    "    # df.append(pd.DataFrame(pool.map(calculate_hits, ziped)))\r\n",
    "    # df.append(pd.DataFrame(pool.map(calculate_neighbour, ziped)))\r\n",
    "    \r\n",
    "    res.append(pd.concat(df, axis=1))\r\n",
    "    del ziped\r\n",
    "    del chunk\r\n",
    "    del df\r\n",
    "    gc.collect()\r\n",
    "    \r\n",
    "graph_features = pd.concat(res)\r\n",
    "graph_features.to_csv('./handled_data/train_feature/train_feature_graph_final.csv', header=None, index=False)\r\n",
    "print(graph_features.shape)\r\n",
    "print(graph_features.describe())\r\n",
    "del graph_features\r\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2BF16F8FC7694CAB9237103C777228F4",
    "collapsed": false,
    "mdEditEnable": false
   },
   "source": [
    "# 一般特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "779E3B5AEB4A4C78893F3321105A4A8C",
    "mdEditEnable": false
   },
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "A34605E7A9914661966CE15A3D173E34",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded dict\nloaded word2vec\nloaded TF-IDF\n----------------initialized models----------------\nCPU times: user 9.07 s, sys: 4.41 s, total: 13.5 s\nWall time: 27.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "dictionary = Dictionary.load('./models/word_dict.dict') # gensim Dictionary\r\n",
    "print('loaded dict')\r\n",
    "w2v_model = KeyedVectors.load('/home/kesci/work/word_vectors/w2v-300d-new.txt')\r\n",
    "print('loaded word2vec')\r\n",
    "tfidf_model = TfidfModel.load('./models/tfidf.model')\r\n",
    "print('loaded TF-IDF')\r\n",
    "fasttext_model = FastText.load('./word_vectors/fast-300d-new.txt')\r\n",
    "# tfidf = joblib.load(model_path + 'scikit_tfidf_model.m')\r\n",
    "print('----------------initialized models----------------')\r\n",
    "\r\n",
    "def init_prob():\r\n",
    "    CHUNK_SIZE = 1000000\r\n",
    "    reader = pd.read_csv('./split/10kw.csv',\r\n",
    "                    chunksize=CHUNK_SIZE,\r\n",
    "                    header=None,\r\n",
    "                    names=['query_id','query','query_title_id','title','label'])\r\n",
    "        \r\n",
    "    totalCounter = collections.Counter()\r\n",
    "    posCounter = collections.Counter()\r\n",
    "    \r\n",
    "    idx = 0\r\n",
    "    for chunk in reader:\r\n",
    "        # if idx == 10: break\r\n",
    "        print(idx)\r\n",
    "        idx += 1\r\n",
    "        query = chunk['query'].apply(lambda x : x.split()).values.tolist()\r\n",
    "        title = chunk['title'].apply(lambda x : x.split()).values.tolist()\r\n",
    "        label = chunk['label'].values.tolist()\r\n",
    "        for i in range(CHUNK_SIZE):\r\n",
    "            for word in query[i]:\r\n",
    "                totalCounter[word] += 1\r\n",
    "                if label[i] == 1:\r\n",
    "                    posCounter[word] += 1\r\n",
    "            for word in title[i]:\r\n",
    "                totalCounter[word] += 1\r\n",
    "                if label[i] == 1:\r\n",
    "                    posCounter[word] += 1\r\n",
    "    prob = {}\r\n",
    "    for key, value in posCounter.items():\r\n",
    "        prob[key] = value / totalCounter[key]\r\n",
    "    return prob   \r\n",
    "# pos_prob = init_prob() # 需要计算出这个单词出现之后label为1的概率\r\n",
    "# file = open(handled_path + 'pos_10kw.pkl', 'wb+')\r\n",
    "# pickle.dump(pos_prob, file)\r\n",
    "\r\n",
    "# file = open('./handled_data/pos_10kw.pkl', 'rb+')\r\n",
    "# pos_prob = pickle.load(file)\r\n",
    "# print('----------------initialized prob----------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "37F857D3CDA0457187A536D291796771",
    "mdEditEnable": false
   },
   "source": [
    "## powerful word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "9A946F0C3D1F408C851722616112EFFA",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.1 s, sys: 204 ms, total: 2.3 s\nWall time: 2.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "# words_power = {}\r\n",
    "# def parse(row):\r\n",
    "#     label = int(row[2])\r\n",
    "#     q1_words = row[0]\r\n",
    "#     q2_words = row[1]\r\n",
    "#     q1_words = set(q1_words)\r\n",
    "#     q2_words = set(q2_words)\r\n",
    "#     all_words = q1_words |q2_words\r\n",
    "#     for word in all_words:\r\n",
    "#         if word not in words_power:\r\n",
    "#             words_power[word] = [0. for i in range(5)]\r\n",
    "#         words_power[word][0] += 1.\r\n",
    "\r\n",
    "#         if ((word in q1_words) and (word not in q2_words)) or ((word not in q1_words) and (word in q2_words)):\r\n",
    "#             words_power[word][1] += 1.\r\n",
    "#             if 1 == label:\r\n",
    "#                 words_power[word][2] += 1.\r\n",
    "#         if (word in q1_words) and (word in q2_words):\r\n",
    "#             words_power[word][3] += 1.\r\n",
    "#             if 1 == label:\r\n",
    "#                 words_power[word][4] += 1.\r\n",
    "# def generate_powerful_word(filepath):\r\n",
    "#     data = pd.read_csv(filepath, chunksize=1000000, header=None)\r\n",
    "\r\n",
    "#     for chunk in tqdm(data):\r\n",
    "#         ziped = list(zip(chunk[1].apply(lambda x:x.split()).tolist(),\r\n",
    "#                         chunk[3].apply(lambda x:x.split()).tolist(),\r\n",
    "#                             chunk[4]))\r\n",
    "#         for row in ziped:\r\n",
    "#             parse(row)\r\n",
    "# generate_powerful_word('/home/kesci/input/bytedance/train_final.csv')\r\n",
    "# for word in tqdm(words_power):\r\n",
    "#     if words_power[word][1] > 1e-6:\r\n",
    "#         words_power[word][2] /= words_power[word][1]\r\n",
    "#     words_power[word][1] /= words_power[word][0]\r\n",
    "#     if words_power[word][3] > 1e-6:\r\n",
    "#         words_power[word][4] /= words_power[word][3]\r\n",
    "#     words_power[word][3] /= words_power[word][0]\r\n",
    "# pickle.dump(words_power, open('./handled_data/powerful_words', 'wb'))\r\n",
    "\r\n",
    "\r\n",
    "words_power = pickle.load(open('./handled_data/powerful_words', 'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7DB87679506D455689343D2DA8A6BD78",
    "mdEditEnable": false
   },
   "source": [
    "## 并行统计特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "9984EF24EE164E1E9DBAC075C899F2DF",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 获取 sentence 的tfidf值\r\n",
    "def getTFIDF(words):\r\n",
    "    tfidf = tfidf_model[dictionary.doc2bow(words)]\r\n",
    "    dic = {word:0 for word in words}\r\n",
    "    sum_weight = 1e-9\r\n",
    "    for idx, val in tfidf:\r\n",
    "        dic[dictionary[idx]] = val\r\n",
    "        sum_weight += val\r\n",
    "        \r\n",
    "    return dic, sum_weight\r\n",
    "\r\n",
    "# query与title都出现过的word的词数除以总词数\r\n",
    "def concurrence(data):\r\n",
    "    query = data[0]\r\n",
    "    title = data[1]\r\n",
    "    query_words = {}\r\n",
    "    title_words = {}\r\n",
    "    for word in query:\r\n",
    "        query_words[word] = query_words.get(word, 0) + 1\r\n",
    "    for word in title:\r\n",
    "        title_words[word] = title_words.get(word, 0) + 1\r\n",
    "\r\n",
    "    shared_query_word = sum([query_words[w] for w in query if w in title])\r\n",
    "    shared_titel_word = sum([title_words[w] for w in title if w in query])\r\n",
    "    total = sum(query_words.values()) + sum(title_words.values())\r\n",
    "\r\n",
    "    if 1e-6 > total:\r\n",
    "        return [0]\r\n",
    "    else:\r\n",
    "        return [1.0 * (shared_titel_word + shared_query_word) / total]\r\n",
    "\r\n",
    "# 编辑距离，词粒度\r\n",
    "def levenshteinDistance(data):\r\n",
    "    query = data[0]\r\n",
    "    title = data[1]\r\n",
    "    \r\n",
    "    len_query = len(query) + 1\r\n",
    "    len_title = len(title) + 1\r\n",
    "    dp = [[0] * len_title] * len_query\r\n",
    "\r\n",
    "    for i in range(1, len_query):\r\n",
    "        for j in range(1, len_title):\r\n",
    "            deletion = dp[i-1][j] + 1\r\n",
    "            insertion = dp[i][j-1] + 1\r\n",
    "            substitution = dp[i-1][j-1]\r\n",
    "            if query[i-1] != title[j-1]:\r\n",
    "                substitution += 1\r\n",
    "            dp[i][j] = min(deletion, insertion, substitution)\r\n",
    "    return [1 - dp[-1][-1] / max(len_title, len_query)]\r\n",
    "\r\n",
    "# sorensen距离\r\n",
    "def sorensenDistance(data):\r\n",
    "    query = data[0]\r\n",
    "    title = data[1]\r\n",
    "    words = set(query) | set(title)\r\n",
    "\r\n",
    "    query_dict = {}\r\n",
    "    for word in query:\r\n",
    "        query_dict[word] = query_dict.get(word, 0) + 1\r\n",
    "\r\n",
    "    title_dict = {}\r\n",
    "    for word in title:\r\n",
    "        title_dict[word] = title_dict.get(word, 0) + 1\r\n",
    "\r\n",
    "    total = sum(query_dict.values()) + sum(title_dict.values())\r\n",
    "\r\n",
    "    diff = 0\r\n",
    "    for word in words:\r\n",
    "        diff += abs(query_dict.get(word, 0) - title_dict.get(word, 0))\r\n",
    "\r\n",
    "    return [diff / total]\r\n",
    "    \r\n",
    "# query和title中的共现词 \r\n",
    "# 计算某个词出现的时候label为1的概率，然后求所有词的概率乘积\r\n",
    "def sameWord(data):\r\n",
    "    unique_words = set(data[0]) | set(data[1])\r\n",
    "    prob = 1\r\n",
    "    for word in unique_words:\r\n",
    "        prob *= pos_prob.get(word, 1)\r\n",
    "    return [len(unique_words), prob]\r\n",
    "\r\n",
    "# Dice Ochi\r\n",
    "# 1 - 交集除以并集jaccard\r\n",
    "def distance(data):\r\n",
    "    query = data[0]\r\n",
    "    title = data[1]\r\n",
    "    intersection = len([x for x in query if x in title])\r\n",
    "    l = len(query) + len(title)\r\n",
    "    return [2*intersection / l, intersection / np.sqrt(l),\r\n",
    "            1- (intersection / (l - intersection))]\r\n",
    "            \r\n",
    "def fuzzyDistance(data):\r\n",
    "    return [\r\n",
    "        fuzz.ratio(data[0], data[1]),\r\n",
    "        fuzz.partial_ratio(data[0], data[1]),\r\n",
    "        fuzz.token_sort_ratio(data[0], data[1]),\r\n",
    "        fuzz.token_set_ratio(data[0], data[1]),\r\n",
    "        fuzz.partial_token_sort_ratio(data[0], data[1]),\r\n",
    "        w2v_model.wmdistance(data[0], data[1]),\r\n",
    "        fasttext_model.wmdistance(data[0], data[1])\r\n",
    "    ]\r\n",
    "\r\n",
    "def powerfuleWord(data):\r\n",
    "    rate_single = 1.0\r\n",
    "    rate_double = 1.0\r\n",
    "    query = set(data[0])\r\n",
    "    title = set(data[1])\r\n",
    "    \r\n",
    "    share_words = query.intersection(title)\r\n",
    "    all_diff = set(query.difference(title) | title.difference(query))\r\n",
    "    for word in share_words:\r\n",
    "        if word in words_power:\r\n",
    "            rate_double *= (1.0 - words_power[word][4])\r\n",
    "    for word in all_diff:\r\n",
    "        if word in words_power:\r\n",
    "            rate_single *= (1.0 - words_power[word][2])\r\n",
    "    \r\n",
    "    return [1-rate_single, 1-rate_double]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3199EF41D500495FB28DFC5C113BDE38",
    "mdEditEnable": false
   },
   "source": [
    "## 并行词嵌入特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "E3A82A356CF04F0986A888EDE46FA47B",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 词向量距离\r\n",
    "def word2vecDistance(data):\r\n",
    "    query = data[0]\r\n",
    "    title = data[1]\r\n",
    "    vec_query = np.array([w2v_model[x] if x in w2v_model else [1e-9]*300 for x in query]).mean(axis=0)\r\n",
    "    vec_title = np.array([w2v_model[x] if x in w2v_model else [1e-9]*300 for x in title]).mean(axis=0)\r\n",
    "\r\n",
    "    cos_dis = 1 - cosine(vec_title, vec_query)\r\n",
    "    euclidean_dis = 1 - euclidean(vec_title, vec_query)\r\n",
    "    manhattan_dis = 1 - cityblock(vec_title, vec_query)\r\n",
    "    \r\n",
    "    vec_query = np.array([fasttext_model[x] if x in fasttext_model else [1e-9]*300 for x in query]).mean(axis=0)\r\n",
    "    vec_title = np.array([fasttext_model[x] if x in fasttext_model else [1e-9]*300 for x in title]).mean(axis=0)\r\n",
    "\r\n",
    "    cos_dis_fasttext = 1 - cosine(vec_title, vec_query)\r\n",
    "    euclidean_dis_fasttext = 1 - euclidean(vec_title, vec_query)\r\n",
    "    manhattan_dis_fasttext = 1 - cityblock(vec_title, vec_query)\r\n",
    "\r\n",
    "    return [cos_dis, euclidean_dis, manhattan_dis, \r\n",
    "            cos_dis_fasttext, euclidean_dis_fasttext, manhattan_dis_fasttext] \r\n",
    "\r\n",
    "# 词向量距离，考虑TFIDF距离\r\n",
    "# 获取重要程度在前topN的词的向量，求距离\r\n",
    "# 只考虑tfidf权重的值的距离，cos、总和、平均\r\n",
    "def w2vWeightDistance(data):\r\n",
    "    topN = 3\r\n",
    "    query = data[0]\r\n",
    "    title = data[1]\r\n",
    "    query_tfidf, query_weight = getTFIDF(query)\r\n",
    "    title_tfidf, title_weight = getTFIDF(title)\r\n",
    "    \r\n",
    "    # word2vec距离，考虑TFIDF\r\n",
    "    vec_query = np.sum(np.array([w2v_model[key] * value if key in w2v_model else [1e-9]*300 for key, value in query_tfidf.items()]), axis=0) / query_weight\r\n",
    "    vec_title = np.sum(np.array([w2v_model[key] * value if key in w2v_model else [1e-9]*300 for key, value in title_tfidf.items()]), axis=0) / title_weight\r\n",
    "\r\n",
    "    cos_dis = 1 - cosine(vec_query, vec_title)\r\n",
    "    if np.isnan(cos_dis):\r\n",
    "        cos_dis = 1\r\n",
    "    euclidean_dis = 1 - euclidean(vec_query, vec_title)\r\n",
    "    manhattan_dis = 1- cityblock(vec_query, vec_title)\r\n",
    "    \r\n",
    "    # fasttext距离，考虑TFIDF\r\n",
    "    vec_query = np.sum(np.array([fasttext_model[key] * value if key in fasttext_model else [1e-9]*300 for key, value in query_tfidf.items()]), axis=0) / query_weight\r\n",
    "    vec_title = np.sum(np.array([fasttext_model[key] * value if key in fasttext_model else [1e-9]*300 for key, value in title_tfidf.items()]), axis=0) / title_weight\r\n",
    "\r\n",
    "    cos_dis_fasttext = 1 - cosine(vec_query, vec_title)\r\n",
    "    if np.isnan(cos_dis_fasttext):\r\n",
    "        cos_dis_fasttext = 1\r\n",
    "    euclidean_dis_fasttext = 1 - euclidean(vec_query, vec_title)\r\n",
    "    manhattan_dis_fasttext = 1- cityblock(vec_query, vec_title)\r\n",
    "    \r\n",
    "    # 获取重要程度在前topN的词\r\n",
    "    query_keywords = sorted(zip(query_tfidf.keys(), query_tfidf.values()), key=lambda x : -x[1])[0 : min(topN, len(query_tfidf))]\r\n",
    "    title_keywords = sorted(zip(title_tfidf.keys(), title_tfidf.values()), key=lambda x : -x[1])[0: min(topN, len(title_tfidf))]\r\n",
    "    \r\n",
    "    # 重要词的word2vec距离\r\n",
    "    query_vector = np.mean(np.array([w2v_model[word] if word in w2v_model else [1e-9]*300 for word, value in query_keywords]), axis=0)\r\n",
    "    title_vector = np.mean(np.array([w2v_model[word] if word in w2v_model else [1e-9]*300 for word, value in title_keywords]), axis=0)\r\n",
    "\r\n",
    "    cos_dis_keyword = 1 - cosine(query_vector, title_vector)\r\n",
    "    \r\n",
    "    euclidean_dis_keyword = 1 - euclidean(query_vector, title_vector)\r\n",
    "    manhattan_dis_keyword = 1 - cityblock(query_vector, title_vector)\r\n",
    "    \r\n",
    "    # 重要词的fasttext距离\r\n",
    "    query_vector = np.mean(np.array([fasttext_model[word] if word in fasttext_model else [1e-9]*300 for word, value in query_keywords]), axis=0)\r\n",
    "    title_vector = np.mean(np.array([fasttext_model[word] if word in fasttext_model else [1e-9]*300 for word, value in title_keywords]), axis=0)\r\n",
    "\r\n",
    "    cos_dis_keyword_fasttext = 1 - cosine(query_vector, title_vector)\r\n",
    "    euclidean_dis_keyword_fasttext = 1 - euclidean(query_vector, title_vector)\r\n",
    "    manhattan_dis_keyword_fasttext = 1 - cityblock(query_vector, title_vector)\r\n",
    "    \r\n",
    "    # 只考虑tfidf权重的值的距离，cosine、sum、avg\r\n",
    "    len1 = query_tfidf.__len__()\r\n",
    "    len2 = title_tfidf.__len__()\r\n",
    "    \r\n",
    "    query_tfidf_value = np.array(list(query_tfidf.values()) + [1e-9]*max(len2-len1, 0))\r\n",
    "    title_tfidf_value = np.array(list(title_tfidf.values()) + [1e-9]*max(len1-len2, 0))\r\n",
    "    \r\n",
    "    tfidf_distance = 1 - cosine(query_tfidf_value, title_tfidf_value)\r\n",
    "    if np.isnan(tfidf_distance):\r\n",
    "        tfidf_distance = 1\r\n",
    "    query_tfidf_sum = np.sum(query_tfidf_value)\r\n",
    "    title_tfidf_sum = np.sum(title_tfidf_value)\r\n",
    "    query_tfidf_mean = np.mean(query_tfidf_value)\r\n",
    "    title_tfidf_mean = np.mean(title_tfidf_value)\r\n",
    "\r\n",
    "    return [cos_dis, euclidean_dis, manhattan_dis, \r\n",
    "            cos_dis_keyword, euclidean_dis_keyword, manhattan_dis_keyword,\r\n",
    "            tfidf_distance, query_tfidf_sum, title_tfidf_sum, \r\n",
    "            query_tfidf_mean, title_tfidf_mean, \r\n",
    "            cos_dis_fasttext, euclidean_dis_fasttext, manhattan_dis_fasttext,\r\n",
    "            cos_dis_keyword_fasttext, euclidean_dis_keyword_fasttext, manhattan_dis_keyword_fasttext]\r\n",
    "\r\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6A7CD35225FD4AF5AC8C590334845120",
    "mdEditEnable": false
   },
   "source": [
    "## 并行生成特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "D74C4E247697433E8555DFB34B7CE243",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "30it [02:33,  5.08s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                  0             1\ncount  3.000000e+07  3.000000e+07\nmean   6.411826e-01  6.411826e-01\nstd    7.225124e-02  7.225124e-02\nmin    0.000000e+00  0.000000e+00\n25%    5.931235e-01  5.931235e-01\n50%    6.340817e-01  6.340817e-01\n75%    6.841270e-01  6.841270e-01\nmax    1.000000e+00  1.000000e+00\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "pool = Pool(8)\r\n",
    "CHUNK_SIZE = 1000000\r\n",
    "data = pd.read_csv('./split/3kw.csv', chunksize=CHUNK_SIZE, header=None)\r\n",
    "data.columns = {'query_id', 'query', 'title_id', 'title', 'label'}\r\n",
    "\r\n",
    "result = []\r\n",
    "i = 0\r\n",
    "for chunk in tqdm(data):\r\n",
    "    i += 1\r\n",
    "    # if i>10:break\r\n",
    "    \r\n",
    "    df = []\r\n",
    "    # query = chunk[1].apply(lambda x : x.split()).tolist()\r\n",
    "    # title = chunk[3].apply(lambda x : x.split()).tolist()\r\n",
    "    \r\n",
    "    query = chunk[1].tolist()\r\n",
    "    title = chunk[3].tolist()\r\n",
    "    d = list(zip(query, title))\r\n",
    "    \r\n",
    "    # df.append(pd.DataFrame(pool.map(concurrence, d)))\r\n",
    "    # print(1)\r\n",
    "    # df.append(pd.DataFrame(pool.map(levenshteinDistance, d)))\r\n",
    "    # print(2)\r\n",
    "    # df.append(pd.DataFrame(pool.map(sorensenDistance, d)))\r\n",
    "    # print(3)\r\n",
    "    # df.append(pd.DataFrame(pool.map(sameWord, d)))\r\n",
    "    # print(4)\r\n",
    "    # df.append(pd.DataFrame(pool.map(distance, d)))\r\n",
    "    # print(5)\r\n",
    "    # df.append(pd.DataFrame(pool.map(word2vecDistance, d)))\r\n",
    "    # print(6)\r\n",
    "    # df.append(pd.DataFrame(pool.map(w2vWeightDistance, d)))\r\n",
    "    # print(7)\r\n",
    "    # df.append(pd.DataFrame(pool.map(fuzzyDistance, d)))\r\n",
    "    # print(8)\r\n",
    "    # df.append(pd.DataFrame(pool.map(powerfuleWord, d)))\r\n",
    "    # print(9)\r\n",
    "    \r\n",
    "    result.append(pd.concat(df, axis=1))\r\n",
    "\r\n",
    "    del df\r\n",
    "    del query\r\n",
    "    del title\r\n",
    "    del d\r\n",
    "    del chunk\r\n",
    "    gc.collect()\r\n",
    "\r\n",
    "del data\r\n",
    "result = pd.concat(result)\r\n",
    "print(result.describe())\r\n",
    "result.to_csv('./handled_data/train_feature/train_feature_jellyfish.csv', header=None, index=False)\r\n",
    "# 释放资源\r\n",
    "pool.close()\r\n",
    "pool.terminate()\r\n",
    "pool.join()\r\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D23CD913318A48C9A86AD2CC28FB1F08",
    "mdEditEnable": false
   },
   "source": [
    "# 特征相关性分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "55A6DF5EEEDE4EB388356B43F0B03CC2",
    "mdEditEnable": false
   },
   "source": [
    "## 相关系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "AC1339E852414CAC9D6C44CA2B2B9C9B",
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9776392403538859 sorensenDistance jaccardDistance\n0.9191483643445837 diceDistance ochiaiDistance\n0.9191483643445837 ochiaiDistance diceDistance\n0.9776392403538859 jaccardDistance sorensenDistance\n0.9790698526721849 word2vecDistance_1 fasttextDistance_1\n0.9013797584783118 word2vecDistance_1 w2vWeightDistance_1\n0.9982673082219394 word2vecDistance_2 word2vecDistance_3\n0.971311990796078 word2vecDistance_2 fasttextDistance_2\n0.9694247650601884 word2vecDistance_2 fasttextDistance_3\n0.9982673082219394 word2vecDistance_3 word2vecDistance_2\n0.9696246774668986 word2vecDistance_3 fasttextDistance_2\n0.9677387984890892 word2vecDistance_3 fasttextDistance_3\n0.9790698526721849 fasttextDistance_1 word2vecDistance_1\n0.971311990796078 fasttextDistance_2 word2vecDistance_2\n0.9696246774668986 fasttextDistance_2 word2vecDistance_3\n0.9980665751664842 fasttextDistance_2 fasttextDistance_3\n0.9694247650601884 fasttextDistance_3 word2vecDistance_2\n0.9677387984890892 fasttextDistance_3 word2vecDistance_3\n0.9980665751664842 fasttextDistance_3 fasttextDistance_2\n0.9013797584783118 w2vWeightDistance_1 word2vecDistance_1\n0.9517133422175729 w2vWeightDistance_1 fasttextWeightDistance_1\n0.9920940986091317 w2vWeightDistance_2 w2vWeightDistance_3\n0.9920940986091317 w2vWeightDistance_3 w2vWeightDistance_2\n0.9822766542630752 w2vKeywordDistance_1 fasttextKeywordDistance_1\n0.9980499767731188 w2vKeywordDistance_2 w2vKeywordDistance_3\n0.9704524215324315 w2vKeywordDistance_2 fasttextKeywordDistance_2\n0.9684116753038846 w2vKeywordDistance_2 fasttextKeywordDistance_3\n0.9980499767731188 w2vKeywordDistance_3 w2vKeywordDistance_2\n0.9685512758429773 w2vKeywordDistance_3 fasttextKeywordDistance_2\n0.9665154030013232 w2vKeywordDistance_3 fasttextKeywordDistance_3\n0.9517133422175729 fasttextWeightDistance_1 w2vWeightDistance_1\n0.9977510526794808 fasttextWeightDistance_2 fasttextWeightDistance_3\n0.9977510526794808 fasttextWeightDistance_3 fasttextWeightDistance_2\n0.9822766542630752 fasttextKeywordDistance_1 w2vKeywordDistance_1\n0.9041843257841926 fasttextKeywordDistance_1 fasttextKeywordDistance_2\n0.9023746325133278 fasttextKeywordDistance_1 fasttextKeywordDistance_3\n0.9704524215324315 fasttextKeywordDistance_2 w2vKeywordDistance_2\n0.9685512758429773 fasttextKeywordDistance_2 w2vKeywordDistance_3\n0.9041843257841926 fasttextKeywordDistance_2 fasttextKeywordDistance_1\n0.9979225827100741 fasttextKeywordDistance_2 fasttextKeywordDistance_3\n0.9684116753038846 fasttextKeywordDistance_3 w2vKeywordDistance_2\n0.9665154030013232 fasttextKeywordDistance_3 w2vKeywordDistance_3\n0.9023746325133278 fasttextKeywordDistance_3 fasttextKeywordDistance_1\n0.9979225827100741 fasttextKeywordDistance_3 fasttextKeywordDistance_2\n0.9999576749537433 pagerank_feature1 seq2_max_degree\n0.9999999866735351 pagerank_feature1 seq1_neighbor\n0.9953337614918548 pagerank_feature1 total_neighbor\n0.9670942889451213 pagerank+feature2 seq1_max_component\n0.9999995206910431 pagerank+feature2 seq2_neighbor\n0.9999576749537433 seq2_max_degree pagerank_feature1\n0.9999576834960514 seq2_max_degree seq1_neighbor\n0.995232077474844 seq2_max_degree total_neighbor\n0.9670942889451213 seq1_max_component pagerank+feature2\n0.9670951938053971 seq1_max_component seq2_neighbor\n0.9999999866735351 seq1_neighbor pagerank_feature1\n0.9999576834960514 seq1_neighbor seq2_max_degree\n0.9953338453527495 seq1_neighbor total_neighbor\n0.9999995206910431 seq2_neighbor pagerank+feature2\n0.9670951938053971 seq2_neighbor seq1_max_component\n0.9953337614918548 total_neighbor pagerank_feature1\n0.995232077474844 total_neighbor seq2_max_degree\n0.9953338453527495 total_neighbor seq1_neighbor\n"
     ]
    }
   ],
   "source": [
    "feature_name = ['concurrence', 'levenshteinDistance', 'sorensenDistance', \n",
    "        'sameWord', 'specialConcurrence',\n",
    "        'diceDistance', 'ochiaiDistance', 'jaccardDistance', \n",
    "        'word2vecDistance_1', 'word2vecDistance_2', 'word2vecDistance_3', \n",
    "        'fasttextDistance_1', 'fasttextDistance_2', 'fasttextDistance_3', \n",
    "        'w2vWeightDistance_1', 'w2vWeightDistance_2', 'w2vWeightDistance_3',\n",
    "        'w2vKeywordDistance_1', 'w2vKeywordDistance_2', 'w2vKeywordDistance_3',\n",
    "        'TFIDFDistance_1', 'TFIDFDistance_2', 'TFIDFDistance_3', \n",
    "        'TFIDFDistance_4', 'TFIDFDistance_5', \n",
    "        'fasttextWeightDistance_1', 'fasttextWeightDistance_2', 'fasttextWeightDistance_3',\n",
    "        'fasttextKeywordDistance_1', 'fasttextKeywordDistance_2', 'fasttextKeywordDistance_3',\n",
    "        'pagerank_feature1', 'pagerank_feature2',\n",
    "        'seq1_max_degree', 'seq2_max_degree', 'seq1_max_component',\n",
    "        'seq1_neighbor', 'seq2_neighbor', 'total_neighbor',\n",
    "        'DSSM_Feature', 'Deep_Model1', 'ARC', 'MVLSTM']\n",
    "        \n",
    "with open('./handled_data/correlation', 'r', encoding='utf-8') as f:\n",
    "    for idx, line in enumerate(f.readlines()):\n",
    "        tmp_data = list(map(lambda x:float(x), line.split(',')))\n",
    "        for item in range(len(tmp_data)):\n",
    "            if tmp_data[item] > 0.9 and item != idx:\n",
    "                print(tmp_data[item], feature_name[idx], feature_name[item])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "08DA2FEA1043479495F8624EAC6E8336",
    "mdEditEnable": false
   },
   "source": [
    "## heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "7C7598F567B7418785EBE3B64A30B1B2",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://cdn.kesci.com/rt_upload/7C7598F567B7418785EBE3B64A30B1B2/pvtej76qjv.png\">"
      ],
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 3000x2000 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_corr():\r\n",
    "    corr = pd.read_csv('./handled_data/correlation', header=None)\r\n",
    "    mask = np.zeros_like(corr, dtype=np.bool)\r\n",
    "    mask[np.triu_indices_from(mask)] = True\r\n",
    "    cmap = sns.diverging_palette(220, 10, as_cmap=True)\r\n",
    "    g = sns.heatmap(corr, mask=mask, cmap=cmap, square=True, annot=True, fmt='0.2f')\r\n",
    "    plt.figure(dpi=500)\r\n",
    "    plt.show()\r\n",
    "    plt.savefig('./try_test')\r\n",
    "plot_corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C4D670551EDD4DF78453E4AEF3437963",
    "mdEditEnable": false
   },
   "source": [
    "# RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "C143F1CF75C84E6C8CB9A410E4982CC7",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del x_train\n",
    "del y_train\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "CA9F515F3FFC4EA488778D87C5268F1C",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------start train-------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=15)]: Using backend ThreadingBackend with 15 concurrent workers.\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(n_jobs=15,\n",
    "                                n_estimators=150,\n",
    "                                class_weight={0:1, 1:3},\n",
    "                                verbose=1)\n",
    "train_x = np.array(x_train.values).reshape(-1, 1).astype('int')\n",
    "train_y = np.array(y_train.values).reshape(-1, 1).astype('int')\n",
    "\n",
    "test_x = np.array(x_test.values).reshape(-1, 1).astype('int')\n",
    "test_y = np.array(y_test.values).reshape(-1, 1).astype('int')\n",
    "print('-------------------start train-------------------')\n",
    "rf.fit(train_x, train_y)\n",
    "pred_rf = rf.predict_proba(test_x)\n",
    "auc_score = roc_auc_score(test_y, pred_rf)\n",
    "acc_score = precision_score(test_y, pred_rf)\n",
    "print('auc_score:', auc_score)\n",
    "print('acc_score:', acc_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "00668DC1B0E74253AC74E3DA5E444CC4",
    "mdEditEnable": false
   },
   "source": [
    "# XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "FD68F203FD1A444486493FB70F4A64D3",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "print(train.get_label())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "BC7D52A783334956A93B90A77C182F38",
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[14:20:06] Tree method is automatically selected to be 'approx' for faster speed. To use old behavior (exact greedy algorithm on single machine), set tree_method to 'exact'.\n"
     ]
    },
    {
     "ename": "XGBoostError",
     "evalue": "b'[14:20:06] /workspace/src/objective/regression_obj.cu:65: Check failed: info.labels_.Size() != 0U (0 vs. 0) label set cannot be empty\\n\\nStack trace returned 10 entries:\\n[bt] (0) /opt/conda/xgboost/libxgboost.so(dmlc::StackTrace()+0x3d) [0x7fb9897f85cd]\\n[bt] (1) /opt/conda/xgboost/libxgboost.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x18) [0x7fb9897f89c8]\\n[bt] (2) /opt/conda/xgboost/libxgboost.so(xgboost::obj::RegLossObj<xgboost::obj::LogisticClassification>::GetGradient(xgboost::HostDeviceVector<float> const&, xgboost::MetaInfo const&, int, xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*)+0xcd) [0x7fb9899ee28d]\\n[bt] (3) /opt/conda/xgboost/libxgboost.so(xgboost::LearnerImpl::UpdateOneIter(int, xgboost::DMatrix*)+0x362) [0x7fb98986f1e2]\\n[bt] (4) /opt/conda/xgboost/libxgboost.so(XGBoosterUpdateOneIter+0x35) [0x7fb9897f0ab5]\\n[bt] (5) /opt/conda/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call_unix64+0x4c) [0x7fb9e0c32ec0]\\n[bt] (6) /opt/conda/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call+0x22d) [0x7fb9e0c3287d]\\n[bt] (7) /opt/conda/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(_ctypes_callproc+0x2ce) [0x7fb9e0e47dee]\\n[bt] (8) /opt/conda/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(+0x12825) [0x7fb9e0e48825]\\n[bt] (9) python(_PyObject_FastCallDict+0x8b) [0x55ebba14c1bb]\\n\\n'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mXGBoostError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xgboost/training.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[1;32m    214\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 216\u001b[0;31m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[1;32m    217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xgboost/training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[0;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[1;32m     72\u001b[0m         \u001b[0;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m             \u001b[0mbst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xgboost/core.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[1;32m   1043\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1044\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[0;32m-> 1045\u001b[0;31m                                                     dtrain.handle))\n\u001b[0m\u001b[1;32m   1046\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1047\u001b[0m             \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xgboost/core.py\u001b[0m in \u001b[0;36m_check_call\u001b[0;34m(ret)\u001b[0m\n\u001b[1;32m    163\u001b[0m     \"\"\"\n\u001b[1;32m    164\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mret\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mXGBoostError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_LIB\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mXGBGetLastError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mXGBoostError\u001b[0m: b'[14:20:06] /workspace/src/objective/regression_obj.cu:65: Check failed: info.labels_.Size() != 0U (0 vs. 0) label set cannot be empty\\n\\nStack trace returned 10 entries:\\n[bt] (0) /opt/conda/xgboost/libxgboost.so(dmlc::StackTrace()+0x3d) [0x7fb9897f85cd]\\n[bt] (1) /opt/conda/xgboost/libxgboost.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x18) [0x7fb9897f89c8]\\n[bt] (2) /opt/conda/xgboost/libxgboost.so(xgboost::obj::RegLossObj<xgboost::obj::LogisticClassification>::GetGradient(xgboost::HostDeviceVector<float> const&, xgboost::MetaInfo const&, int, xgboost::HostDeviceVector<xgboost::detail::GradientPairInternal<float> >*)+0xcd) [0x7fb9899ee28d]\\n[bt] (3) /opt/conda/xgboost/libxgboost.so(xgboost::LearnerImpl::UpdateOneIter(int, xgboost::DMatrix*)+0x362) [0x7fb98986f1e2]\\n[bt] (4) /opt/conda/xgboost/libxgboost.so(XGBoosterUpdateOneIter+0x35) [0x7fb9897f0ab5]\\n[bt] (5) /opt/conda/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call_unix64+0x4c) [0x7fb9e0c32ec0]\\n[bt] (6) /opt/conda/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call+0x22d) [0x7fb9e0c3287d]\\n[bt] (7) /opt/conda/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(_ctypes_callproc+0x2ce) [0x7fb9e0e47dee]\\n[bt] (8) /opt/conda/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(+0x12825) [0x7fb9e0e48825]\\n[bt] (9) python(_PyObject_FastCallDict+0x8b) [0x55ebba14c1bb]\\n\\n'"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "%%time\n",
    "# train, test = train_test_split(total, test_size=0.1, random_state=20, stratify=total[label_idx])\n",
    "# print('end split')\n",
    "# param = {'objective': 'binary:logistic'}\n",
    "# train = xgb.DMatrix(train)\n",
    "# test = xgb.DMatrix(test)\n",
    "# print(train.num_col())\n",
    "# print(train.num_row())\n",
    "\n",
    "bst = xgb.train(param, train, 2)\n",
    "\n",
    "# pred_train = bst.predict(train)\n",
    "# acc_score = accuracy_score(train.get_label(), pred_train)\n",
    "# print(acc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "77F7C13F377544128FEE0CC87859FAE9",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del total\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "63DA97D8ADDE475EACA2EB7875FE2A26"
   },
   "outputs": [],
   "source": [
    "pred_test = bst.predict(test)\n",
    "test_acc = accuracy_score(test.get_label(), pred_test)\n",
    "test_auc = roc_auc_score(test.get_label(), pred_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D90F0E908A3E4E4A8436C11BA5CA0537",
    "mdEditEnable": false
   },
   "source": [
    "# LightGBM模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6ADF6DA536D94C5B99CA1A9D2C0FE965",
    "mdEditEnable": false
   },
   "source": [
    "## 准备训练数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "4371E4A6693B4712A3FFE7FCB3B96050",
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-11 19:19:30.037143\nloaded normal feature     ---> (30000000, 31)\nloaded graph feature      ---> (30000000, 5)\nloaded fuzzy feature      ---> (30000000, 7)\nloaded powerful feature   ---> (30000000, 2)\nloaded dssm_300 feature   ---> (30000000, 1)\nloaded dssm_600 feature   ---> (30000000, 1)\nloaded mvlstm feature     ---> (30000000, 1)\nloaded ARC feature        ---> (30000000, 1)\nloaded TextCNN feature    ---> (30000000, 1)\nloaded OOF CNN feature    ---> (30000000, 1)\nloaded OOF LSTM feature   ---> (30000000, 1)\nloaded label with shape   ---> (30000000, 1)\n----------------------end reading file--------------------------\ntotal feature shape       ---> (30000000, 53)\nsplit train set and test set\ninitialize dataset\nCPU times: user 8min 10s, sys: 40.9 s, total: 8min 51s\nWall time: 9min 9s\n"
     ]
    }
   ],
   "source": [
    "%%time\r\n",
    "\r\n",
    "print(datetime.now())\r\n",
    "# 一般特征\r\n",
    "feature_normal = pd.read_csv('./handled_data/train_feature/train_feature_3kw_normal.csv.gz', header=None)\r\n",
    "print('loaded normal feature     --->',feature_normal.shape)\r\n",
    "\r\n",
    "# feature_statistics = pd.read_csv('./handled_data/train_feature/train_feature_3kw_statistics.csv.gz', header=None)\r\n",
    "# print('loaded statistics feature --->',feature_statistics.shape)\r\n",
    "# feature_pagerank = pd.read_csv('./handled_data/train_feature/train_feature_3kw_pagerank.csv.gz', header=None)\r\n",
    "# print('loaded pagerank feature   --->',feature_pagerank.shape)\r\n",
    "\r\n",
    "feature_graph = pd.read_csv('./handled_data/train_feature/train_feature_graph_final.csv.gz', header=None)\r\n",
    "print('loaded graph feature      --->',feature_graph.shape)\r\n",
    "\r\n",
    "feature_fuzzy = pd.read_csv('./handled_data/train_feature/train_feature_fuzzy.csv', header=None)\r\n",
    "print('loaded fuzzy feature      --->',feature_fuzzy.shape)\r\n",
    "feature_powerful = pd.read_csv('./handled_data/train_feature/train_feature_powerful.csv', header=None)\r\n",
    "print('loaded powerful feature   --->',feature_powerful.shape)\r\n",
    "\r\n",
    "# 融合特征\r\n",
    "feat_dssm_300 = pd.read_csv('./predictions/feat_dssm_300_3kw.csv.gz', header=None)\r\n",
    "print('loaded dssm_300 feature   --->', feat_dssm_300.shape)\r\n",
    "\r\n",
    "feat_dssm_600 = pd.read_csv('./predictions/feat_dssm_600_3kw.csv.gz', header=None)\r\n",
    "print('loaded dssm_600 feature   --->', feat_dssm_600.shape)   \r\n",
    "\r\n",
    "feat_mvlstm = pd.read_csv('./predictions/feat_mvlstm_3kw.csv.gz', header=None)\r\n",
    "print('loaded mvlstm feature     --->', feat_mvlstm.shape) \r\n",
    "\r\n",
    "feat_arc = pd.read_csv('./predictions/feat_arc_3kw.csv.gz', header=None)\r\n",
    "print('loaded ARC feature        --->', feat_arc.shape) \r\n",
    "    \r\n",
    "feat_textcnn = pd.read_csv('./predictions/feat_textcnn_3kw.csv.gz', header=None)\r\n",
    "print('loaded TextCNN feature    --->', feat_textcnn.shape)\r\n",
    "\r\n",
    "feat_oofcnn = pd.read_csv('./predictions/feat_oofcnn_3kw.csv', header=None)\r\n",
    "print('loaded OOF CNN feature    --->', feat_oofcnn.shape)\r\n",
    "\r\n",
    "feat_ooflstm = pd.read_csv('./predictions/feat_ooflstm_3kw.csv', header=None)\r\n",
    "print('loaded OOF LSTM feature   --->', feat_ooflstm.shape)\r\n",
    "\r\n",
    "label = pd.read_csv('./handled_data/label', header=None)\r\n",
    "print('loaded label with shape   --->',label.shape)\r\n",
    "print('----------------------end reading file--------------------------')\r\n",
    "total = pd.concat([feature_normal, \r\n",
    "                    # feature_statistics, feature_pagerank, \r\n",
    "                    feature_graph,\r\n",
    "                    feature_fuzzy, feature_powerful,\r\n",
    "                    feat_dssm_300,\r\n",
    "                    feat_dssm_600,\r\n",
    "                    feat_mvlstm,\r\n",
    "                    feat_arc,\r\n",
    "                    feat_textcnn,\r\n",
    "                    feat_oofcnn,\r\n",
    "                    feat_ooflstm,\r\n",
    "                    label], axis=1)\r\n",
    "total.columns = range(total.shape[1])\r\n",
    "print('total feature shape       --->', total.shape)\r\n",
    "\r\n",
    "print('split train set and test set')\r\n",
    "label_idx = total.shape[1] - 1\r\n",
    "\r\n",
    "train, val = train_test_split(total, test_size=0.1, random_state=20, stratify=total[label_idx])\r\n",
    "\r\n",
    "x = train.drop(label_idx, axis=1)\r\n",
    "y = train[label_idx]\r\n",
    "\r\n",
    "val_x = val.drop(label_idx, axis=1)\r\n",
    "val_y = val[label_idx]\r\n",
    "\r\n",
    "print('initialize dataset')\r\n",
    "lgb_train = lgb.Dataset(x, y, free_raw_data=False)\r\n",
    "lgb_eval = lgb.Dataset(val_x, val_y, reference=lgb_train, free_raw_data=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "019B00D9B9914A51A76009D3FBA3E7C5",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "total_x = total.drop(label_idx, axis=1)\n",
    "total_y = total[label_idx]\n",
    "print(total_x.shape)\n",
    "print(total_x.info())\n",
    "print(total_y.shape)\n",
    "pritn(total_y.info())\n",
    "lgb_train = lgb.Dataset(total_x, total_y, free_raw_data=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "D79357D486D244678838A702C3148530",
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000000, 52)\n                 0             1             2             3              4   \\\ncount  3.000000e+07  3.000000e+07  3.000000e+07  3.000000e+07   3.000000e+07   \nmean   3.138709e-01  6.360163e-01  7.210984e-01  1.328846e+01   5.023723e-05   \nstd    1.871304e-01  1.825601e-01  1.542859e-01  4.734423e+00   7.655814e-04   \nmin    0.000000e+00  3.225806e-02  0.000000e+00  1.000000e+00  3.876599e-302   \n25%    1.904762e-01  5.000000e-01  6.363636e-01  1.000000e+01   1.425644e-12   \n50%    2.857143e-01  6.666667e-01  7.333333e-01  1.300000e+01   3.036743e-10   \n75%    4.090909e-01  7.777778e-01  8.181818e-01  1.600000e+01   6.644035e-08   \nmax    3.902353e+01  1.000000e+00  1.000000e+00  3.850000e+02   6.000000e-01   \n\n                 5             6             7             8             9   \\\ncount  3.000000e+07  3.000000e+07  3.000000e+07  3.000000e+07  3.000000e+07   \nmean   2.809463e-01  5.420982e-01  8.260438e-01  7.305404e-01 -7.067164e-01   \nstd    1.559342e-01  2.922543e-01  1.176581e-01  1.138556e-01  4.590602e-01   \nmin    0.000000e+00  0.000000e+00 -5.142857e+00 -1.806917e-01 -5.323372e+00   \n25%    1.818182e-01  3.333333e-01  7.777778e-01  6.662080e-01 -9.964515e-01   \n50%    2.666667e-01  5.163978e-01  8.461538e-01  7.450130e-01 -6.658506e-01   \n75%    3.636364e-01  7.071068e-01  9.000000e-01  8.105919e-01 -3.794953e-01   \nmax    1.720000e+00  1.576659e+01  1.000000e+00  1.000000e+00  1.000000e+00   \n\n           ...                 42            43            44            45  \\\ncount      ...       3.000000e+07  3.000000e+07  3.000000e+07  3.000000e+07   \nmean       ...       3.053099e+00  8.375073e-01  3.500157e-01  4.725512e-01   \nstd        ...       6.043442e-01  1.453110e-01  1.638842e-01  1.583299e-01   \nmin        ...       0.000000e+00  0.000000e+00  0.000000e+00 -4.724671e-01   \n25%        ...       2.714013e+00  7.789553e-01  2.111369e-01  3.822839e-01   \n50%        ...       3.127549e+00  8.863827e-01  3.517495e-01  4.979405e-01   \n75%        ...       3.467873e+00  9.420421e-01  4.681469e-01  5.879889e-01   \nmax        ...       5.794093e+00  1.000000e+00  1.000000e+00  8.932437e-01   \n\n                 46            47            48            49            50  \\\ncount  3.000000e+07  3.000000e+07  3.000000e+07  3.000000e+07  3.000000e+07   \nmean   4.913790e-01  1.704771e-01  1.771518e-01  1.857886e-01  1.686962e-01   \nstd    1.603338e-01  4.920971e-02  3.686785e-02  5.073331e-02  4.917796e-02   \nmin   -4.589797e-01  1.637830e-02  1.085761e-02  7.413505e-03  2.439348e-03   \n25%    3.987670e-01  1.346706e-01  1.530675e-01  1.492922e-01  1.368189e-01   \n50%    5.176989e-01  1.684497e-01  1.779618e-01  1.904999e-01  1.739967e-01   \n75%    6.093000e-01  2.041900e-01  2.028155e-01  2.268609e-01  2.051853e-01   \nmax    9.266943e-01  3.532417e-01  3.169906e-01  2.812516e-01  5.599475e-01   \n\n                 51  \ncount  3.000000e+07  \nmean   1.833208e-01  \nstd    6.467187e-02  \nmin    2.731318e-14  \n25%    1.404598e-01  \n50%    1.873902e-01  \n75%    2.247325e-01  \nmax    9.985868e-01  \n\n[8 rows x 52 columns]\n(30000000,)\ncount    3.000000e+07\nmean     1.822721e-01\nstd      3.860687e-01\nmin      0.000000e+00\n25%      0.000000e+00\n50%      0.000000e+00\n75%      0.000000e+00\nmax      1.000000e+00\nName: 52, dtype: float64\n"
     ]
    }
   ],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "55C87C30E20844FA8677C3BBFD4E2D45",
    "mdEditEnable": false
   },
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "0716C46A15684CA1A77343249DC9FAD1",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-11 19:36:41.955449\nCPU times: user 9h 24min 49s, sys: 1min 7s, total: 9h 25min 56s\nWall time: 41min 38s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<lightgbm.basic.Booster at 0x7f9562d70a20>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\r\n",
    "params = {\r\n",
    "    'boosting': 'dart',\r\n",
    "    'objective': 'binary',\r\n",
    "    'metric': 'auc',\r\n",
    "    'learning_rate': '0.5',\r\n",
    "    \r\n",
    "    'num_leaves':59,\r\n",
    "    'max_depth':7,\r\n",
    "    \r\n",
    "    'max_bin':251,\r\n",
    "    'min_data_in_leaf':19,\r\n",
    "         \r\n",
    "    'scale_pos_weight': 3.1,\r\n",
    "         \r\n",
    "    'lambda_l1':0,\r\n",
    "    'lambda_l2':0,\r\n",
    "    'min_split_gain':0,\r\n",
    "    \r\n",
    "    'device': 'gpu',\r\n",
    "    'gpu_platform_id':0,\r\n",
    "    'gpu_device_id':0\r\n",
    "}\r\n",
    "\r\n",
    "print(datetime.now())\r\n",
    "gbm = lgb.train(params, lgb_train, num_boost_round=450, early_stopping_rounds=30)\r\n",
    "gbm.save_model('./handled_data/final_lgb_450.model') \r\n",
    "# init_model=gbm, keep_training_booster=True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4B62C670C46546168FAA5DB184641EB9",
    "mdEditEnable": false
   },
   "source": [
    "## GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "182A5DC4B3D74AC195675B4D444AFCEA",
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 21 candidates, totalling 105 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.\n[Parallel(n_jobs=10)]: Done  30 tasks      | elapsed: 34.0min\n[Parallel(n_jobs=10)]: Done 105 out of 105 | elapsed: 120.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'min_data_in_leaf': 251} \n 0.7039860009576545\n"
     ]
    }
   ],
   "source": [
    "# 6 64\r\n",
    "# 7 58\r\n",
    "# params = {'max_depth':range(4,8), 'num_leaves':range(56, 60)}\r\n",
    "# 2 20\r\n",
    "# params = {'min_child_samples':range(18,23)}\r\n",
    "\r\n",
    "# params ={'max_bin': range(5,100,10), }\r\n",
    "\r\n",
    "params = {'min_data_in_leaf':range(240, 261)}\r\n",
    "gbm = lgb.LGBMClassifier(boost='dart',\r\n",
    "                            max_depth=7, \r\n",
    "                            num_leaves=59,\r\n",
    "                            max_bin = 95,\r\n",
    "                            learning_rate=0.5, \r\n",
    "                            n_estimators=100,\r\n",
    "                            metric='auc')\r\n",
    "gridSearch = GridSearchCV(estimator=gbm, param_grid=params, scoring='roc_auc', \r\n",
    "                                    cv=5, verbose=1, n_jobs=15)\r\n",
    "\r\n",
    "gridSearch.fit(x, y)\r\n",
    "\r\n",
    "print(gridSearch.best_params_ , '\\n', gridSearch.best_score_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EDEF130C7D6E4C149C34CA5FA916CA2C",
    "mdEditEnable": false
   },
   "source": [
    "## 查看特征重要性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "AB78DAF736034D958BD510FBBDD13A49",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52\n52\n                 feature_name  importance\n0                    DSSM_300        1522\n1             seq1_max_degree        1460\n2           pagerank_feature2        1374\n3                     fuzzy_1        1246\n4           pagerank_feature1        1182\n5                     fuzzy_3        1154\n6                     fuzzy_4        1119\n7                  doubleSide         890\n8                    sameWord         750\n9                      OOFCNN         743\n10         fasttextDistance_1         611\n11        levenshteinDistance         590\n12                    fuzzy_5         562\n13            TFIDFDistance_2         550\n14         specialConcurrence         519\n15                   DSSM_600         512\n16                    fuzzy_2         476\n17         word2vecDistance_1         448\n18            seq2_max_degree         430\n19   fasttextWeightDistance_2         425\n20                        ARC         399\n21   fasttextWeightDistance_1         397\n22                 singleSide         392\n23                   word2vec         360\n24             ochiaiDistance         340\n25                    OOFLSTM         328\n26        w2vWeightDistance_1         316\n27  fasttextKeywordDistance_1         307\n28                    TextCNN         296\n29            TFIDFDistance_3         281\n30                     MVLSTM         254\n31           sorensenDistance         244\n32                concurrence         229\n33        w2vWeightDistance_2         222\n34            TFIDFDistance_4         219\n35   fasttextWeightDistance_3         204\n36       w2vKeywordDistance_1         191\n37  fasttextKeywordDistance_2         185\n38            TFIDFDistance_1         173\n39            TFIDFDistance_5         173\n40         word2vecDistance_2         162\n41        w2vWeightDistance_3         149\n42         word2vecDistance_3         142\n43  fasttextKeywordDistance_3         133\n44                   fasttext         120\n45            jaccardDistance         119\n46         fasttextDistance_3         112\n47       w2vKeywordDistance_2         109\n48       w2vKeywordDistance_3         102\n49         fasttextDistance_2          94\n50         seq1_max_component          85\n51               diceDistance          71\n0 concurrence 229\n1 levenshteinDistance 590\n2 sorensenDistance 244\n3 sameWord 750\n4 specialConcurrence 519\n5 diceDistance 71\n6 ochiaiDistance 340\n7 jaccardDistance 119\n8 word2vecDistance_1 448\n9 word2vecDistance_2 162\n10 word2vecDistance_3 142\n11 fasttextDistance_1 611\n12 fasttextDistance_2 94\n13 fasttextDistance_3 112\n14 w2vWeightDistance_1 316\n15 w2vWeightDistance_2 222\n16 w2vWeightDistance_3 149\n17 w2vKeywordDistance_1 191\n18 w2vKeywordDistance_2 109\n19 w2vKeywordDistance_3 102\n20 TFIDFDistance_1 173\n21 TFIDFDistance_2 550\n22 TFIDFDistance_3 281\n23 TFIDFDistance_4 219\n24 TFIDFDistance_5 173\n25 fasttextWeightDistance_1 397\n26 fasttextWeightDistance_2 425\n27 fasttextWeightDistance_3 204\n28 fasttextKeywordDistance_1 307\n29 fasttextKeywordDistance_2 185\n30 fasttextKeywordDistance_3 133\n31 seq1_max_degree 1460\n32 seq2_max_degree 430\n33 seq1_max_component 85\n34 pagerank_feature1 1182\n35 pagerank_feature2 1374\n36 fuzzy_1 1246\n37 fuzzy_2 476\n38 fuzzy_3 1154\n39 fuzzy_4 1119\n40 fuzzy_5 562\n41 word2vec 360\n42 fasttext 120\n43 singleSide 392\n44 doubleSide 890\n45 DSSM_300 1522\n46 DSSM_600 512\n47 MVLSTM 254\n48 ARC 399\n49 TextCNN 296\n50 OOFCNN 743\n51 OOFLSTM 328\n"
     ]
    }
   ],
   "source": [
    "name = ['concurrence', 'levenshteinDistance', 'sorensenDistance', \r\n",
    "        'sameWord', 'specialConcurrence',\r\n",
    "        'diceDistance', 'ochiaiDistance', 'jaccardDistance', \r\n",
    "        'word2vecDistance_1', 'word2vecDistance_2', 'word2vecDistance_3', \r\n",
    "        'fasttextDistance_1', 'fasttextDistance_2', 'fasttextDistance_3', \r\n",
    "        'w2vWeightDistance_1', 'w2vWeightDistance_2', 'w2vWeightDistance_3',\r\n",
    "        'w2vKeywordDistance_1', 'w2vKeywordDistance_2', 'w2vKeywordDistance_3',\r\n",
    "        'TFIDFDistance_1', 'TFIDFDistance_2', 'TFIDFDistance_3', \r\n",
    "        'TFIDFDistance_4', 'TFIDFDistance_5', \r\n",
    "        'fasttextWeightDistance_1', 'fasttextWeightDistance_2', 'fasttextWeightDistance_3',\r\n",
    "        'fasttextKeywordDistance_1', 'fasttextKeywordDistance_2', 'fasttextKeywordDistance_3',\r\n",
    "        'seq1_max_degree', 'seq2_max_degree', 'seq1_max_component',\r\n",
    "        'pagerank_feature1', 'pagerank_feature2',\r\n",
    "        'fuzzy_1', 'fuzzy_2', 'fuzzy_3', 'fuzzy_4', 'fuzzy_5', 'word2vec', 'fasttext',\r\n",
    "        'singleSide', 'doubleSide',\r\n",
    "        # 'seq1_neighbor', 'seq2_neighbor', 'total_neighbor',\r\n",
    "        'DSSM_300', 'DSSM_600', \r\n",
    "        'MVLSTM', 'ARC','TextCNN','OOFCNN' ,'OOFLSTM' ]\r\n",
    "        \r\n",
    "print(len(name))\r\n",
    "print(gbm.feature_importance().__len__())\r\n",
    "feature_importance = pd.DataFrame({'feature_name': name[:gbm.feature_importance().__len__()], \r\n",
    "                                'importance': gbm.feature_importance()}).sort_values(by='importance', ascending=False).reset_index(drop = True)\r\n",
    "print(feature_importance)\r\n",
    "for i in range(len(gbm.feature_importance())):\r\n",
    "    print(i, name[i], gbm.feature_importance()[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D54F3E9A7A8244968747F2C45CF15329",
    "mdEditEnable": false
   },
   "source": [
    "## 测试结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "70BD1ABB77634747A35495A26496A518",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-11 20:18:38.544223\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'val_x' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'val_x' is not defined"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "%%time\n",
    "print(datetime.now())\n",
    "# gbm = lgb.Booster(model_file='./handled_data/lgb_2kw_650.model')\n",
    "pred = gbm.predict(val_x, num_iteration=gbm.best_iteration)\n",
    "\n",
    "print('end predict')\n",
    "result = []\n",
    "for item in pred:\n",
    "    if item>=0.5:\n",
    "        result.append(1)\n",
    "    else:\n",
    "        result.append(0)\n",
    "count = 0\n",
    "pos_count = 0\n",
    "print('pos_predict:',np.sum(result))\n",
    "\n",
    "val_y_list = val_y.tolist()\n",
    "for i in range(len(pred)):\n",
    "    if val_y_list[i] == 1 and result[i] == val_y_list[i]:\n",
    "        pos_count += 1\n",
    "    if result[i] == val_y_list[i]:\n",
    "        count+=1\n",
    "print('acc:',count/len(pred))\n",
    "print('pos_right_count:',pos_count)\n",
    "print('pos_label:',np.sum(val_y.tolist()))\n",
    "print('pos_acc:',pos_count / np.sum(val_y.tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "A3E2CEF3CFC641CC9E2A3CD628F7C161",
    "mdEditEnable": false
   },
   "source": [
    "## 提交文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "952FBCFDA61D40D6B19562CB54E2575D",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "LightGBMError",
     "evalue": "Could not open ./handled_data/lgb_power_450.model",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mLightGBMError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, params, train_set, model_file, model_str, silent)\u001b[0m\n\u001b[1;32m   1662\u001b[0m                 \u001b[0mc_str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1663\u001b[0m                 \u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbyref\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_num_iterations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1664\u001b[0;31m                 ctypes.byref(self.handle)))\n\u001b[0m\u001b[1;32m   1665\u001b[0m             \u001b[0mout_num_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_int\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1666\u001b[0m             _safe_call(_LIB.LGBM_BoosterGetNumClasses(\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36m_safe_call\u001b[0;34m(ret)\u001b[0m\n\u001b[1;32m     45\u001b[0m     \"\"\"\n\u001b[1;32m     46\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mret\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mLightGBMError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdecode_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_LIB\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLGBM_GetLastError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mLightGBMError\u001b[0m: Could not open ./handled_data/lgb_power_450.model"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "gbm = lgb.Booster(model_file='./handled_data/final_lgb_450.model')\n",
    "feature_normal_test = pd.read_csv('./handled_data/test_feature/test_feature_3kw_normal.csv', header=None)\n",
    "print('loaded feature normal     --->',feature_normal_test.shape)\n",
    "feature_statistics_test = pd.read_csv('./handled_data/test_feature/test_feature_3kw_statistics.csv', header=None)\n",
    "print('loaded feature statistics --->',feature_statistics_test.shape)\n",
    "feature_pagerank_test = pd.read_csv('./handled_data/test_feature/test_feature_3kw_pagerank.csv', header=None)\n",
    "print('loaded feature pagerank   --->',feature_pagerank_test.shape)\n",
    "feature_fuzzy_test = pd.read_csv('./handled_data/test_feature/test_feature_fuzzy.csv', header=None)\n",
    "print('loaded fuzzy feature      --->',feature_fuzzy_test.shape)\n",
    "feature_powerful_test = pd.read_csv('./handled_data/test_feature/test_feature_powerful.csv', header=None)\n",
    "print('loaded powerfule feature  --->',feature_powerful_test.shape)\n",
    "\n",
    "# 融合特征\n",
    "feat_dssm_300_test = pd.read_csv('./predictions/test_dssm_300_2kw.csv.gz', header=None)\n",
    "print('loaded feature dssm_300   --->', feat_dssm_300_test.shape)\n",
    "feat_dssm_600_test = pd.read_csv('./predictions/test_dssm_600_2kw.csv.gz', header=None)\n",
    "print('loaded feature dssm_600   --->', feat_dssm_600_test.shape)\n",
    "feat_mvlstm_test = pd.read_csv('./predictions/test_mv_2kw.csv.gz', header=None)\n",
    "print('loaded feature MVLSTM     --->', feat_mvlstm_test.shape)\n",
    "feat_arc_test = pd.read_csv('./predictions/test_arc_2kw.csv.gz', header=None)\n",
    "print('loaded feature ARC        --->', feat_arc_test.shape)\n",
    "feat_textcnn_test = pd.read_csv('./predictions/test_textcnn_2kw.csv.gz', header=None)\n",
    "print('loaded feature TextCNN    --->', feat_textcnn_test.shape)\n",
    "\n",
    "# feat_lstm_dssm_test = pd.read_csv('./predictions/test_lstm_dssm_2kw.csv', header=None)\n",
    "# print('loaded feature LSTM-DSSM  --->', feat_lstm_dssm_test.shape)\n",
    "\n",
    "print('------------finish reading file, concating feature------------')\n",
    "test_feature = pd.concat([feature_normal_test, feature_statistics_test, feature_pagerank_test, \n",
    "                feature_fuzzy_test, feature_powerful_test, \n",
    "                feat_dssm_300_test,\n",
    "                feat_dssm_600_test,\n",
    "                feat_mvlstm_test,\n",
    "                feat_arc_test,\n",
    "                feat_textcnn_test,\n",
    "                # feat_lstm_dssm_test\n",
    "                ], axis=1)\n",
    "print('concated feature, shape    --->', test_feature.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "C709A47F9B8F45119C4BB2DEF2F96AFA",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish prediction, shpae   ---> (20000000, 3)\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "pred_test = pd.DataFrame(gbm.predict(test_feature, num_iteration=gbm.best_iteration))\n",
    "\n",
    "test_data = pd.read_csv(test_data_path, header=None)\n",
    "submission = pd.concat([test_data[0], test_data[2], pred_test], axis=1)\n",
    "print('finish prediction, shpae   --->', submission.shape)\n",
    "submission.to_csv('./predictions/lgb_submission_power_450.csv', header=None, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "B5882F60E2E9433285E7C3C4B65EC5D7",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kesci Submit Tool 3.2.1\n\n> 已验证Token\n> 提交文件 ./predictions/lgb_submission_power_450.csv (565534.77 KiB)\n> 已上传 100 %\n> 文件已上传        \n> 服务器响应: 200 提交成功，请等待评审完成\n> 提交完成\n"
     ]
    }
   ],
   "source": [
    "!https_proxy=\"http://klab-external-proxy\" ./kesci_submit -file ./predictions/lgb_submission_power_450.csv -token 8be4f72dc2395a8d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BA927732609B435D981ABD77FF0A00E3",
    "mdEditEnable": false
   },
   "source": [
    "## Final提交文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "166E19CE673F447A84B2C4EC610BB4B5",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# !wc -l /home/kesci/work/predictions/testfin_mvlstm.csv\n",
    "# !wc -l /home/kesci/work/predictions/testfin_textcnn.csv\n",
    "!killall python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "FE7C8E4888A14AAA81B133E6F0F80CD4",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[       1        2        3 ... 10318080 10318081 10318082]\n[       1        2        3 ... 10318080 10318081 10318082]\n10318082\n10318082\n\n[ 1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20]\n[ 1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20]\n20\n20\nCPU times: user 8.47 s, sys: 2.29 s, total: 10.8 s\nWall time: 11 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# final = pd.read_csv(final_data_path, header=None)\n",
    "# result = pd.read_csv('./predictions/final_submission_20190811.csv', header=None)\n",
    "print(final[0].unique())\n",
    "print(result[0].unique())\n",
    "print(final[0].unique().__len__())\n",
    "print(result[0].unique().__len__())\n",
    "print()\n",
    "\n",
    "print(final[2].unique())\n",
    "print(result[1].unique())\n",
    "print(final[2].unique().__len__())\n",
    "print(result[1].unique().__len__())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "84E58641682D4B9BB34BA28DA4F058BF",
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r  0%|          | 0/10 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-08-11 20:20:04.433063\nstart iteration\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r 10%|█         | 1/10 [04:20<39:01, 260.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r 20%|██        | 2/10 [08:31<34:19, 257.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r 30%|███       | 3/10 [12:46<29:57, 256.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r 40%|████      | 4/10 [16:55<25:27, 254.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r 50%|█████     | 5/10 [21:05<21:06, 253.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r 60%|██████    | 6/10 [25:20<16:54, 253.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r 70%|███████   | 7/10 [29:33<12:40, 253.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r 80%|████████  | 8/10 [33:45<08:26, 253.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r 90%|█████████ | 9/10 [37:57<04:12, 252.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r100%|██████████| 10/10 [42:09<00:00, 252.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000000, 3)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                  0             1             2\ncount  1.000000e+08  1.000000e+08  1.000000e+08\nmean   5.170643e+06  7.224337e+00  4.130191e-01\nstd    2.967558e+06  5.240233e+00  1.687013e-01\nmin    1.000000e+00  1.000000e+00  0.000000e+00\n25%    2.606632e+06  3.000000e+00  2.880982e-01\n50%    5.195137e+06  6.000000e+00  4.040706e-01\n75%    7.727133e+06  1.100000e+01  5.317806e-01\nmax    1.031808e+07  2.000000e+01  1.000000e+00\nfinish prediction, shpae   ---> (100000000, 3)\nCPU times: user 2h 26min 37s, sys: 4min 11s, total: 2h 30min 49s\nWall time: 50min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(datetime.now())\n",
    "CHUNKSIZE = 10000000\n",
    "gbm = lgb.Booster(model_file='./handled_data/final_lgb_450.model')\n",
    "\n",
    "feature_normal_final = pd.read_csv('./handled_data/final/final_feature_normal.csv', header=None, chunksize=CHUNKSIZE)\n",
    "feature_statistics_final = pd.read_csv('./handled_data/final/final_feature_statistics.csv', header=None, chunksize=CHUNKSIZE)\n",
    "feature_pagerank_final = pd.read_csv('./handled_data/final/final_feature_pagerank.csv', header=None, chunksize=CHUNKSIZE)\n",
    "feature_fuzzy_final = pd.read_csv('./handled_data/final/final_feature_fuzzy.csv', header=None, chunksize=CHUNKSIZE)\n",
    "feature_powerful_final = pd.read_csv('./handled_data/final/final_feature_powerful.csv', header=None, chunksize=CHUNKSIZE)\n",
    "\n",
    "feat_dssm_300_final = pd.read_csv('./predictions/testfin_300_dssm.csv_10kw.gz', header=None, chunksize=CHUNKSIZE)\n",
    "feat_dssm_600_final = pd.read_csv('./predictions/testfin_600_dssm.csv', header=None, chunksize=CHUNKSIZE)\n",
    "feat_mvlstm_final = pd.read_csv('./predictions/testfin_mvlstm.csv', header=None, chunksize=CHUNKSIZE)\n",
    "feat_arc_final = pd.read_csv('./predictions/testfin_arc.csv', header=None, chunksize=CHUNKSIZE)\n",
    "feat_textcnn_final = pd.read_csv('./predictions/testfin_textcnn.csv', header=None, chunksize=CHUNKSIZE)\n",
    "feat_oofcnn_final = pd.read_csv('./predictions/testfin_oofcnn.csv', header=None, chunksize=CHUNKSIZE)\n",
    "feat_ooflstm_final = pd.read_csv('./predictions/testfin_ooflstm.csv', header=None, chunksize=CHUNKSIZE)\n",
    "final_data = pd.read_csv(final_data_path, header=None, chunksize=CHUNKSIZE)\n",
    "result = []\n",
    "print('start iteration')\n",
    "for i in tqdm(range(10)):\n",
    "    features = pd.concat([feature_normal_final.get_chunk(), \n",
    "                            feature_statistics_final.get_chunk(), feature_pagerank_final.get_chunk(),\n",
    "                            feature_fuzzy_final.get_chunk(), feature_powerful_final.get_chunk(),\n",
    "                            feat_dssm_300_final.get_chunk(),\n",
    "                            feat_dssm_600_final.get_chunk(),\n",
    "                            feat_mvlstm_final.get_chunk(),\n",
    "                            feat_arc_final.get_chunk(),\n",
    "                            feat_textcnn_final.get_chunk(),\n",
    "                            feat_oofcnn_final.get_chunk(),\n",
    "                            feat_ooflstm_final.get_chunk()], axis=1)\n",
    "                            \n",
    "    idx = final_data.get_chunk()\n",
    "    pred = gbm.predict(features, num_iteration=gbm.best_iteration)\n",
    "    result.append(pd.DataFrame({'query_id': idx[0].tolist(), 'title_id': idx[2].tolist(), 'label': pred}))\n",
    "    print(result[i].shape)\n",
    "    \n",
    "    del pred\n",
    "    del idx\n",
    "    del features\n",
    "    gc.collect()\n",
    "    \n",
    "result = pd.concat(result)\n",
    "result.columns = range(result.shape[1])\n",
    "print(result.describe())\n",
    "print('finish prediction, shpae   --->', result.shape)\n",
    "result.to_csv('./predictions/final_submission_20190811_2.csv', header=None, index=False)\n",
    "\n",
    "# feature_append_test = pd.read_csv('./handled_data/final/test_feature_append.csv', header=None)\n",
    "# print('loaded append feature     --->',feature_append_test.shape)\n",
    "\n",
    "# 融合特征\n",
    "# with gzip.open('./predictions/testfin_300_dssm.csv_10kw.gz', 'rb') as f:\n",
    "#     feat_dssm_300_final = pd.read_csv(f, header=None)\n",
    "# print('loaded feature dssm_300   --->', feat_dssm_300_final.shape)\n",
    "    \n",
    "# with gzip.open('./predictions/testfin_600_dssm.csv_10kw.gz', 'rb') as f:\n",
    "#     feat_dssm_600_final = pd.read_csv(f, header=None)\n",
    "# feat_dssm_600_final = pd.read_csv('./predictions/testfin_600_dssm_p1.csv', header=None)\n",
    "# print('loaded feature dssm_600   --->', feat_dssm_600_final.shape)\n",
    "    \n",
    "# with gzip.open('./predictions/test_mv_2kw.csv.gz', 'rb') as f:\n",
    "#     feat_mvlstm_test = pd.read_csv(f, header=None)\n",
    "# print('loaded feature MVLSTM     --->', feat_mvlstm_test.shape)\n",
    "    \n",
    "# with gzip.open('./predictions/test_arc_2kw.csv.gz', 'rb') as f:\n",
    "#     feat_arc_test = pd.read_csv(f, header=None)\n",
    "# print('loaded feature ARC        --->', feat_arc_test.shape)\n",
    "    \n",
    "# with gzip.open('./predictions/test_textcnn_2kw.csv.gz', 'rb') as f:\n",
    "#     feat_textcnn_test = pd.read_csv(f, header=None)\n",
    "# print('loaded feature TextCNN    --->', feat_textcnn_test.shape)\n",
    "\n",
    "# feat_lstm_dssm_test = pd.read_csv('./predictions/test_lstm_dssm_2kw.csv', header=None)\n",
    "# print('loaded feature LSTM-DSSM  --->', feat_lstm_dssm_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "D3072559E2AB4D09B7A6B1A5C5F99C5D",
    "hide_input": false,
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kesci Submit Tool 3.2.1\n\n> 已验证Token\n> 提交文件 ./predictions/final_submission_20190811_2.csv (2877771.46 KiB)\n> 已上传 100 %\n> 文件已上传        \n> 服务器响应: 200 提交成功\n> 提交完成\n"
     ]
    }
   ],
   "source": [
    "!https_proxy=\"http://klab-external-proxy\" ./kesci_submit -file ./predictions/final_submission_20190811_2.csv -token 8be4f72dc2395a8d -mode archive            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "F8C1BDC9AC484B3F81EA96EEF49108D9",
    "mdEditEnable": false
   },
   "source": [
    "# end"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  },
  "language_info": {
   "name": "python",
   "version": "3.6.4",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
